266 lines
14 KiB
Python
266 lines
14 KiB
Python
"""Checks of the web app, without calling claude: .venv/bin/python app/server/check.py [name]"""
|
|||
|
|
import json
|
||
|
|
import os
|
||
|
|
import shutil
|
||
|
|
import socket
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
import urllib.request
|
||
|
|
from pathlib import Path
|
||
|
|
from urllib.error import HTTPError
|
||
|
|
|
||
|
|
HERE = Path(__file__).resolve().parent
|
||
|
|
sys.path.insert(0, str(HERE))
|
||
|
|
import uvicorn # noqa: E402
|
||
|
|
import server # noqa: E402
|
||
|
|
import station # noqa: E402
|
||
|
|
|
||
|
|
|
||
|
|
def temp_folder():
|
||
|
|
return Path(tempfile.mkdtemp())
|
||
|
|
|
||
|
|
|
||
|
|
def check_server():
|
||
|
|
# the real server on a free port: the guard (Host, declared JSON), a read
|
||
|
|
with socket.socket() as so:
|
||
|
|
so.bind(("127.0.0.1", 0))
|
||
|
|
port = so.getsockname()[1]
|
||
|
|
srv = uvicorn.Server(uvicorn.Config(server.app, host="127.0.0.1", port=port, log_level="error"))
|
||
|
|
threading.Thread(target=srv.run, daemon=True).start()
|
||
|
|
while not srv.started:
|
||
|
|
time.sleep(0.05)
|
||
|
|
base = f"http://127.0.0.1:{port}"
|
||
|
|
|
||
|
|
def call(path, body=None, headers=None, method=None):
|
||
|
|
r = urllib.request.Request(base + path, json.dumps(body).encode() if body is not None else None,
|
||
|
|
{"Content-Type": "application/json", **(headers or {})}, method=method)
|
||
|
|
try:
|
||
|
|
with urllib.request.urlopen(r) as rep:
|
||
|
|
return rep.status
|
||
|
|
except HTTPError as e:
|
||
|
|
return e.code
|
||
|
|
try:
|
||
|
|
assert call("/api/stations", headers={"Host": "evil.example:80"}) == 403
|
||
|
|
assert call("/api/labels", {"station": "x"}, {"Content-Type": "text/plain"}) == 403
|
||
|
|
assert call("/api/stations") == 200
|
||
|
|
assert call("/api/analyses") == 200
|
||
|
|
assert call("/api/analyses/UNKNOWN/optimize", {}) == 400
|
||
|
|
finally:
|
||
|
|
srv.should_exit = True
|
||
|
|
|
||
|
|
|
||
|
|
def refuses(f, *a):
|
||
|
|
try:
|
||
|
|
f(*a)
|
||
|
|
except station.Refusal as e:
|
||
|
|
return str(e)
|
||
|
|
raise AssertionError(f"{f.__name__}{a} did not refuse")
|
||
|
|
|
||
|
|
|
||
|
|
def check_station():
|
||
|
|
# the data of the Data page, on the real data; the labels in a temporary folder
|
||
|
|
assert "NIEDERHORN" in station.stations()
|
||
|
|
im = station.images("NIEDERHORN", "2026-09-25", "2026-09-26")
|
||
|
|
assert im["images"] and im["positions"] and set(im["corr"]) == set(im["positions"]), list(im)
|
||
|
|
assert all(len(c) == len(im["images"]) for c in im["corr"].values())
|
||
|
|
assert station.image("NIEDERHORN", im["images"][0]["name"]).is_file()
|
||
|
|
refuses(station.image, "NIEDERHORN", "../../../.env")
|
||
|
|
refuses(station.images, "../API", "", "")
|
||
|
|
assert station.images("LENZSPITZE", "2026-10-26", "2026-10-27")["images"]
|
||
|
|
c = station.curves("LENZSPITZE", "2026-10-26", "2026-10-27")
|
||
|
|
titles = [p["title"] for p in c["plots"]]
|
||
|
|
assert "Relative Corrected Vertical Shift" in titles and any(t.startswith("System: ") for t in titles), titles
|
||
|
|
ev = station.detectors("LENZSPITZE", "2026-10-26", "2026-10-27") # the detectors of the journal, ~10 s
|
||
|
|
assert ev and {"start", "end", "family", "description", "sensors"} <= set(ev[0]), ev[:1]
|
||
|
|
assert all(e["message"] and e["action"] and set(e["targets"] + e["references"]) == set(e["sensors"]) for e in ev), ev[:1]
|
||
|
|
assert station.message("A moved; nothing to do. The jump of 2 targets in this period has the same cause.") \
|
||
|
|
== "A moved. The jump of 2 targets in this period has the same cause."
|
||
|
|
assert all(e["end"] >= "2026-10-26T00:00Z" and e["start"] < "2026-10-27T00:00Z" for e in ev), [(e["start"], e["end"]) for e in ev]
|
||
|
|
assert not any("SIM_CCID" in t or t.startswith("System: ip") for t in titles)
|
||
|
|
assert all(len(c["T"][tr["t"]]) == len(tr["y"]) for p in c["plots"] for tr in p["traces"])
|
||
|
|
before, station.LABELS = station.LABELS, temp_folder()
|
||
|
|
try:
|
||
|
|
e = station.add({"station": "NIEDERHORN", "folder": "auto", "start": "2026-09-25T01:00Z",
|
||
|
|
"end": "2026-09-25T02:00Z", "family": "Visibility", "description": "fog"})
|
||
|
|
assert e["id"] == "NIEDERHORN-1" and next(iter(e)) == "id", e
|
||
|
|
e2 = station.update({"station": "NIEDERHORN", "folder": "auto", "before": e,
|
||
|
|
"start": e["start"], "end": e["end"], "family": "", "description": "rain"})
|
||
|
|
assert e2 == {"id": "NIEDERHORN-1", "start": e["start"], "end": e["end"], "description": "rain"}, e2
|
||
|
|
assert station.read("NIEDERHORN", "auto") == [e2]
|
||
|
|
assert "no longer in the file" in refuses(station.delete, {"station": "NIEDERHORN", "folder": "auto", "before": e})
|
||
|
|
station.delete({"station": "NIEDERHORN", "folder": "auto", "before": e2})
|
||
|
|
assert station.read("NIEDERHORN", "auto") == []
|
||
|
|
refuses(station.add, {"station": "NIEDERHORN", "folder": "../x", "start": "a", "end": "b", "description": "x"})
|
||
|
|
refuses(station.add, {"station": "NIEDERHORN", "start": "2026-09-25T02:00Z", "end": "2026-09-25T01:00Z", "description": "x"})
|
||
|
|
finally:
|
||
|
|
shutil.rmtree(station.LABELS)
|
||
|
|
station.LABELS = before
|
||
|
|
|
||
|
|
|
||
|
|
def check_optimization():
|
||
|
|
# the Optimize button (analysis.optimize), without claude: a fake process that ends only on a signal, a finished
|
||
|
|
# analysis copied into a temporary folder; the lock (one optimization at a time, no analysis or message meanwhile), the
|
||
|
|
# state read again after a restart
|
||
|
|
import analysis
|
||
|
|
end = threading.Event()
|
||
|
|
|
||
|
|
launched = []
|
||
|
|
|
||
|
|
class FakeProcess:
|
||
|
|
returncode = 0
|
||
|
|
stdout = iter(lambda: end.wait() and "", "") # nothing, until the signal
|
||
|
|
|
||
|
|
def __init__(self, args, *a, **k):
|
||
|
|
launched.append((args, k))
|
||
|
|
|
||
|
|
def wait(self):
|
||
|
|
return 0
|
||
|
|
|
||
|
|
before = analysis.FOLDER, analysis.subprocess.Popen, analysis.OPTIMIZER, analysis.SESSIONS, analysis.AGENTS
|
||
|
|
key = os.environ.get("ANTHROPIC_API_KEY")
|
||
|
|
os.environ["ANTHROPIC_API_KEY"] = "test-key"
|
||
|
|
root = temp_folder()
|
||
|
|
analysis.FOLDER, analysis.OPTIMIZER, analysis.AGENTS = root / "analyses", root / "optimizer", root / "agents"
|
||
|
|
analysis.SESSIONS = analysis.OPTIMIZER / "sessions"
|
||
|
|
analysis.subprocess.Popen = FakeProcess
|
||
|
|
try:
|
||
|
|
for name in ("NORDEND_a", "NORDEND_b"):
|
||
|
|
(analysis.FOLDER / name / "work").mkdir(parents=True)
|
||
|
|
(analysis.FOLDER / name / "request.json").write_text(json.dumps(
|
||
|
|
{"station": "NORDEND", "start": "2026-11-04T21:12Z", "end": "2026-11-07T14:12Z", "hints": ""}))
|
||
|
|
(analysis.FOLDER / name / "report.md").write_text("# report")
|
||
|
|
assert analysis.state("NORDEND_a")["optimization"] is None
|
||
|
|
# without Optimizer, then without agents: a clear refusal, not an error trace
|
||
|
|
assert "optimizer/" in refuses(analysis.optimize, "NORDEND_a")
|
||
|
|
(analysis.OPTIMIZER / ".claude" / "skills" / "improve").mkdir(parents=True)
|
||
|
|
(analysis.OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").write_text("improve")
|
||
|
|
assert "agents/" in refuses(analysis.optimize, "NORDEND_a")
|
||
|
|
(analysis.AGENTS / ".claude-plugin").mkdir(parents=True)
|
||
|
|
(analysis.AGENTS / ".claude-plugin" / "plugin.json").write_text("{}")
|
||
|
|
assert "optimizer/" in refuses(analysis.optimize, "NORDEND_a") # without the data, labeling, detectors links of Optimizer
|
||
|
|
for link in ("data", "labeling", "detectors"):
|
||
|
|
(analysis.OPTIMIZER / link).mkdir()
|
||
|
|
assert analysis.optimize("NORDEND_a") == {}
|
||
|
|
assert analysis.state("NORDEND_a")["optimization"]["state"] == "running"
|
||
|
|
refuses(analysis.optimize, "NORDEND_a")
|
||
|
|
refuses(analysis.optimize, "NORDEND_b")
|
||
|
|
refuses(analysis.launch, {"station": "NORDEND", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"})
|
||
|
|
refuses(analysis.message, "NORDEND_b", {"text": "continue"})
|
||
|
|
end.set()
|
||
|
|
while analysis.state("NORDEND_a")["optimization"]["state"] == "running":
|
||
|
|
time.sleep(0.05)
|
||
|
|
assert analysis.state("NORDEND_a")["optimization"]["state"] == "done"
|
||
|
|
args, k = launched[0] # /improve in Optimizer, on the range of the analysis, TO excluded
|
||
|
|
request = args[2]
|
||
|
|
assert request.startswith("/improve ") and "STATION: data/NORDEND. FROM: 2026-11-04. TO: 2026-11-08." in request \
|
||
|
|
and "OUT: sessions/NORDEND_a_" in request and "FAMILIES: labeling/families.md." in request \
|
||
|
|
and "CONTEXT: labeling/auto/analyses/NORDEND_a/report.md." in request, request
|
||
|
|
assert k["cwd"] == analysis.OPTIMIZER.resolve() and "--plugin-dir" in args, (args, k)
|
||
|
|
assert all("ANTHROPIC_API_KEY" not in kw["env"] for _, kw in launched), "an API key reaches claude"
|
||
|
|
# isolated from the settings of the workstation (the user's plugins, hooks, MCP servers)
|
||
|
|
assert all(a[a.index("--setting-sources") + 1] == "project,local" and "--strict-mcp-config" in a for a, _ in launched)
|
||
|
|
args, k = launched[1] # then the tool-builder of the agents repository, on the tools repository
|
||
|
|
assert args[args.index("--agent") + 1] == "agents:tool-builder" and "Tools: tools/" in args[2] \
|
||
|
|
and args[2].endswith("/tools.md."), args
|
||
|
|
analysis.analyses["NORDEND_b"]["state"] = "running" # a running analysis blocks the optimization
|
||
|
|
refuses(analysis.optimize, "NORDEND_a")
|
||
|
|
# after a restart: the stream ends with {"type": "end"} -> done; without that line -> interrupted; the summary of
|
||
|
|
# the detectors and that of the tools in one page
|
||
|
|
stream = analysis.FOLDER / "NORDEND_a" / "work" / "optimization.jsonl"
|
||
|
|
assert stream.read_text().endswith('{"type": "end", "state": "done"}\n'), stream.read_text()
|
||
|
|
analysis.analyses.clear()
|
||
|
|
analysis.optimizations.clear()
|
||
|
|
assert analysis.state("NORDEND_a")["optimization"]["state"] == "done"
|
||
|
|
stream.write_text('{"type": "assistant", "message": {"content": []}}\n')
|
||
|
|
analysis.optimizations.clear()
|
||
|
|
assert analysis.state("NORDEND_a")["optimization"]["state"] == "interrupted"
|
||
|
|
(analysis.SESSIONS / "NORDEND_a_20261113T1000Z").mkdir(parents=True)
|
||
|
|
(analysis.SESSIONS / "NORDEND_a_20261113T1000Z" / "summary.md").write_text("# improve\n\n**Result**: none")
|
||
|
|
# a new analysis of the same range (NORDEND_a_143000): its session is not the summary of NORDEND_a
|
||
|
|
(analysis.SESSIONS / "NORDEND_a_143000_20261113T1100Z").mkdir(parents=True)
|
||
|
|
(analysis.SESSIONS / "NORDEND_a_143000_20261113T1100Z" / "summary.md").write_text("# other analysis")
|
||
|
|
assert analysis._summary_file("NORDEND_a").parent.name == "NORDEND_a_20261113T1000Z", analysis._summary_file("NORDEND_a")
|
||
|
|
(analysis.FOLDER / "NORDEND_a" / "tools.md").write_text("# Tools\n\n**Tools changed**: none")
|
||
|
|
assert analysis.state("NORDEND_a")["optimization"]["summary"]
|
||
|
|
page = analysis.summary("NORDEND_a", False)
|
||
|
|
assert "<strong>Result</strong>" in page and "<strong>Tools changed</strong>" in page, page
|
||
|
|
assert analysis.state("NORDEND_b")["optimization"] is None
|
||
|
|
finally:
|
||
|
|
end.set()
|
||
|
|
os.environ.pop("ANTHROPIC_API_KEY")
|
||
|
|
if key is not None:
|
||
|
|
os.environ["ANTHROPIC_API_KEY"] = key
|
||
|
|
shutil.rmtree(root)
|
||
|
|
analysis.FOLDER, analysis.subprocess.Popen, analysis.OPTIMIZER, analysis.SESSIONS, analysis.AGENTS = before
|
||
|
|
analysis.analyses.clear()
|
||
|
|
analysis.optimizations.clear()
|
||
|
|
|
||
|
|
|
||
|
|
def check_analysis():
|
||
|
|
# "Analyze the range" then a message (analysis.launch, analysis.message), without claude: the analyst of the agents
|
||
|
|
# repository, its parameters, and the resumption of its session with the same agent
|
||
|
|
import analysis
|
||
|
|
launched = []
|
||
|
|
|
||
|
|
class FakeProcess:
|
||
|
|
returncode = 0
|
||
|
|
|
||
|
|
def __init__(self, args, *a, **k):
|
||
|
|
launched.append((args, k))
|
||
|
|
self.stdout = iter([])
|
||
|
|
|
||
|
|
def wait(self):
|
||
|
|
return 0
|
||
|
|
|
||
|
|
(station.ROOT / "out").mkdir(exist_ok=True) # under the root: the request cites the output folder relative to it
|
||
|
|
before = analysis.FOLDER, analysis.subprocess.Popen, analysis.submit
|
||
|
|
analysis.FOLDER = Path(tempfile.mkdtemp(dir=station.ROOT / "out"))
|
||
|
|
analysis.subprocess.Popen = FakeProcess
|
||
|
|
analysis.submit = lambda *a: 0 # otherwise the test rewrites the real labeling/auto/NORDEND.jsonl
|
||
|
|
key = os.environ.get("ANTHROPIC_API_KEY")
|
||
|
|
os.environ["ANTHROPIC_API_KEY"] = "test-key" # the agent runs on the subscription, never on an API key
|
||
|
|
try:
|
||
|
|
name = analysis.launch({"station": "NORDEND", "start": "2026-11-05T06:00Z", "end": "2026-11-05T12:00Z"})["name"]
|
||
|
|
while analysis.analyses[name]["state"] == "running":
|
||
|
|
time.sleep(0.05)
|
||
|
|
assert analysis.analyses[name]["state"] == "done", analysis.analyses[name]
|
||
|
|
assert json.loads((analysis.FOLDER / name / "request.json").read_text()) == \
|
||
|
|
{"station": "NORDEND", "start": "2026-11-05T06:00Z", "end": "2026-11-05T12:00Z", "hints": ""}
|
||
|
|
args, k = launched[0]
|
||
|
|
assert args[args.index("--agent") + 1] == "agents:analyst" and "--plugin-dir" in args, args
|
||
|
|
assert "ANTHROPIC_API_KEY" not in k["env"] and "PATH" in k["env"], sorted(k.get("env") or {})
|
||
|
|
assert args[2].startswith("Station: data/NORDEND/. Range: 2026-11-05T06:00Z → 2026-11-05T12:00Z (UTC). Hints: none. ") \
|
||
|
|
and "DETECTORS: no" in args[2] and "INSTRUCTIONS: instructions/analyst.md" in args[2], args[2]
|
||
|
|
assert args[args.index("--setting-sources") + 1] == "project,local" and "--strict-mcp-config" in args, args
|
||
|
|
(analysis.FOLDER / name / "work" / "session").write_text("s-1")
|
||
|
|
analysis.message(name, {"text": "continue"})
|
||
|
|
while analysis.analyses[name]["state"] == "running":
|
||
|
|
time.sleep(0.05)
|
||
|
|
args, k = launched[1]
|
||
|
|
assert "ANTHROPIC_API_KEY" not in k["env"], sorted(k.get("env") or {})
|
||
|
|
assert args[args.index("--resume") + 1] == "s-1" and args[args.index("--agent") + 1] == "agents:analyst" \
|
||
|
|
and "--plugin-dir" in args, args
|
||
|
|
agents, analysis.AGENTS = analysis.AGENTS, analysis.FOLDER / "no-agents" # the agents/ link is missing: clear refusal
|
||
|
|
try:
|
||
|
|
assert "agents/" in refuses(analysis.launch, {"station": "NORDEND", "start": "2026-11-06T06:00Z", "end": "2026-11-06T12:00Z"})
|
||
|
|
finally:
|
||
|
|
analysis.AGENTS = agents
|
||
|
|
finally:
|
||
|
|
os.environ.pop("ANTHROPIC_API_KEY")
|
||
|
|
if key is not None:
|
||
|
|
os.environ["ANTHROPIC_API_KEY"] = key
|
||
|
|
shutil.rmtree(analysis.FOLDER)
|
||
|
|
analysis.FOLDER, analysis.subprocess.Popen, analysis.submit = before
|
||
|
|
analysis.analyses.clear()
|
||
|
|
|
||
|
|
|
||
|
|
CHECKS = [check_server, check_station, check_optimization, check_analysis]
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
name = sys.argv[1] if len(sys.argv) > 1 else None
|
||
|
|
for f in CHECKS:
|
||
|
|
if name in (None, f.__name__):
|
||
|
|
f()
|
||
|
|
print("ok", f.__name__)
|