397 lines
20 KiB
Python
397 lines
20 KiB
Python
"""The cases: the phenomena, false friends and lessons drawn from the labels, one JSON file each in auto/cases/ (the
|
|
design: docs/superpowers/specs/2026-11-18-cases-design.md); the RAG that finds those closest to a text, and the
|
|
bookkeeping of their refresh by the `cases` skill (.claude/skills/cases/). manual/ is the user's alone: no process uses it.
|
|
|
|
The analyst describes what it sees on its range; the tool returns the cases that speak of it most, each with its
|
|
signature, its checks, its neighbours and how to tell them apart, its sources and whether the user validated it: it is
|
|
up to the analyst to judge whether they fit.
|
|
|
|
python3 labeling/scripts/cases.py "the targets jump together when a reference is rejected intermittently"
|
|
python3 labeling/scripts/cases.py "<text>" --family Obstruction --k 5
|
|
python3 labeling/scripts/cases.py --list # every case, one line each, without Ollama
|
|
python3 labeling/scripts/cases.py --index # only updates the index
|
|
python3 labeling/scripts/cases.py --check # checks the logic, without Ollama
|
|
python3 labeling/scripts/cases.py --pending DIR # the refresh: the labels to read, one file per family, in DIR
|
|
python3 labeling/scripts/cases.py --record FILE # the refresh: {"assignments": {label id: [case ids]}} applied
|
|
|
|
auto/cases/.read.json, the build's state: per label id, the hash of the text it was read with and the cases it went to.
|
|
|
|
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed cases go through
|
|
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
|
|
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
|
|
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
|
|
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
|
|
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
|
|
INDEX = HERE / ".index" / f"cases_{MODEL.replace(':', '_')}.json"
|
|
BATCH = 32 # texts per call to the model
|
|
|
|
|
|
def cases(root=HERE):
|
|
"""Every case of auto/cases/ (not the dotfiles, the build's state), sorted by id."""
|
|
out = []
|
|
for f in sorted(glob.glob(str(root / "auto" / "cases" / "*.json"))):
|
|
try:
|
|
out.append(json.loads(Path(f).read_text(encoding="utf-8")))
|
|
except json.JSONDecodeError:
|
|
sys.exit(f"{f}: unreadable JSON")
|
|
return sorted(out, key=lambda c: c["id"])
|
|
|
|
|
|
def labels(root=HERE):
|
|
"""Every label of auto/ by id, with its station; never those of manual/."""
|
|
out = {}
|
|
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
|
|
for line in open(f, encoding="utf-8"):
|
|
if line.strip():
|
|
e = json.loads(line)
|
|
out[e["id"]] = e | {"station": Path(f).stem}
|
|
return out
|
|
|
|
|
|
def provenance(c):
|
|
return f"validated by the user on {c['review']}" if c.get("review") else "not validated"
|
|
|
|
|
|
def text(c):
|
|
return f"{c['kind']}. {c.get('family') or ''}. {c['name']}. {c['signature']}"
|
|
|
|
|
|
def listing(cs):
|
|
return [f"{c['id']} | {c['kind']} | {c.get('family') or '-'} | {c['name']}" for c in cs]
|
|
|
|
|
|
def key(t):
|
|
return hashlib.sha1(t.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def norm(v):
|
|
n = math.sqrt(sum(x * x for x in v)) or 1.0
|
|
return [round(x / n, 5) for x in v]
|
|
|
|
|
|
def start_ollama(wait=120):
|
|
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
|
|
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
|
|
return False
|
|
deadline = time.monotonic() + wait
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
|
|
return True
|
|
except OSError:
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
def embed(texts):
|
|
body = json.dumps({"model": MODEL, "input": texts}).encode()
|
|
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
|
|
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=300) as r:
|
|
return [norm(v) for v in json.load(r)["embeddings"]]
|
|
except OSError as err:
|
|
if attempt == 2 or not start_ollama():
|
|
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
|
|
|
|
|
|
def updated_index(everything):
|
|
"""The vectors of every case; computes the missing ones, forgets those of vanished texts."""
|
|
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
|
|
wanted = {key(text(e)): text(e) for e in everything}
|
|
missing = [k for k in wanted if k not in vectors]
|
|
for i in range(0, len(missing), BATCH):
|
|
batch = missing[i:i + BATCH]
|
|
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
|
|
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
|
|
if missing or len(vectors) != len(wanted):
|
|
vectors = {k: vectors[k] for k in wanted}
|
|
INDEX.parent.mkdir(exist_ok=True)
|
|
INDEX.write_text(json.dumps(vectors))
|
|
return vectors
|
|
|
|
|
|
def search(q, everything, vectors, k=8, family=None):
|
|
"""The k cases closest to the query vector q (of `family` and the cross-family ones, if given) -> [(similarity, case)]."""
|
|
kept = [c for c in everything if not family or c.get("family") in (family, None)]
|
|
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(c))])), c) for c in kept]
|
|
return sorted(scores, key=lambda n: -n[0])[:k]
|
|
|
|
|
|
def main_list(root=HERE):
|
|
for line in listing(cases(root)):
|
|
print(line)
|
|
|
|
|
|
def show(score, c, known):
|
|
"""One case found, in full, its label sources dated from `known` (the labels of auto/ by id)."""
|
|
def source(s):
|
|
e = known.get(s)
|
|
return f"{s} ({e['station']} {e['start']} → {e['end']})" if e else (s if s in ("user", "expert", "client report") else f"{s} (not in auto/)")
|
|
out = [f"## {c['id']} — {c['kind']} — {c.get('family') or 'no family'} — similarity {score:.2f}; {provenance(c)}",
|
|
c["name"], c["signature"]]
|
|
out += [f"check: {x}" for x in c.get("checks") or []]
|
|
out += [f"neighbour {n['case']}: {n['tell_apart']}" for n in c.get("neighbours") or []]
|
|
if c.get("client"):
|
|
out.append(f"client: {c['client']}")
|
|
out.append("sources: " + "; ".join(source(s) for s in c["sources"]))
|
|
return "\n".join(out) + "\n"
|
|
|
|
|
|
def label_hash(label):
|
|
"""What a label says, for the refresh: a label read again when its bounds, family or description change."""
|
|
return key(f"{label['start']} {label['end']} {label['family']}. {label['description']}")
|
|
|
|
|
|
def _read_path(root):
|
|
return root / "auto" / "cases" / ".read.json"
|
|
|
|
|
|
def _write(path, obj):
|
|
"""Writes a JSON file, readable in a diff, next to it then replaces it: an interruption does not damage it."""
|
|
tmp = path.with_name(path.name + ".tmp")
|
|
tmp.write_text(json.dumps(obj, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def pending(root=HERE):
|
|
"""The labels to read, by family (new, changed, or attached to no existing case), and the ids deleted since the
|
|
last record -> ({family: [label]}, [id])."""
|
|
known, existing = labels(root), {c["id"] for c in cases(root)}
|
|
f = _read_path(root)
|
|
read = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {}
|
|
todo = {}
|
|
for i, e in known.items():
|
|
r = read.get(i)
|
|
if not r or r["hash"] != label_hash(e) or not r["cases"] or any(c not in existing for c in r["cases"]):
|
|
todo.setdefault(e["family"], []).append(e)
|
|
return todo, sorted(i for i in read if i not in known)
|
|
|
|
|
|
def write_pending(todo, out):
|
|
"""One file per family in `out`, `<family in lowercase, dashes>.jsonl`, one label per line. Removes first the files
|
|
of an earlier refresh it owns (labels, answers, assignments), and only those: a wrong `out` loses nothing else."""
|
|
from validate import FAMILIES
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
for family in FAMILIES:
|
|
for suffix in (".jsonl", ".answer.json"):
|
|
(out / f"{family.lower().replace(' ', '-')}{suffix}").unlink(missing_ok=True)
|
|
(out / "assignments.json").unlink(missing_ok=True)
|
|
for family, ls in todo.items():
|
|
(out / f"{family.lower().replace(' ', '-')}.jsonl").write_text(
|
|
"".join(json.dumps(e, ensure_ascii=False) + "\n" for e in ls), encoding="utf-8")
|
|
|
|
|
|
def record(assignments, root=HERE):
|
|
"""Applies {label id: [case ids]} (the cases each label read illustrates) to the sources of the cases and to
|
|
.read.json; removes from every case the label sources no longer in auto/; deletes the cases left without a source
|
|
(a non-label source keeps a case). Never touches `review` -> {added, removed, deleted_cases, skipped}."""
|
|
from validate import NON_LABEL
|
|
known = labels(root)
|
|
before = {c["id"]: c for c in cases(root)}
|
|
cs = json.loads(json.dumps(before))
|
|
if unknown := sorted({i for v in assignments.values() for i in v if i not in cs}):
|
|
sys.exit(f"unknown case(s): {unknown}; nothing written")
|
|
f = _read_path(root)
|
|
read = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {}
|
|
added, removed = {}, {}
|
|
skipped = sorted(i for i in assignments if i not in known)
|
|
for c in cs.values():
|
|
for s in [s for s in c["sources"] if s not in NON_LABEL and s not in known]:
|
|
c["sources"].remove(s)
|
|
removed.setdefault(c["id"], []).append(s)
|
|
read = {i: r for i, r in read.items() if i in known}
|
|
for i, new in assignments.items():
|
|
if i in skipped:
|
|
continue
|
|
for c in cs.values():
|
|
if i in c["sources"] and c["id"] not in new:
|
|
c["sources"].remove(i)
|
|
removed.setdefault(c["id"], []).append(i)
|
|
for cid in new:
|
|
if i not in cs[cid]["sources"]:
|
|
cs[cid]["sources"].append(i)
|
|
added.setdefault(cid, []).append(i)
|
|
read[i] = {"hash": label_hash(known[i]), "cases": list(new)}
|
|
deleted = sorted(cid for cid, c in cs.items() if not c["sources"])
|
|
folder = root / "auto" / "cases"
|
|
for cid, c in cs.items():
|
|
if cid in deleted:
|
|
(folder / f"{cid}.json").unlink()
|
|
elif c != before[cid]:
|
|
_write(folder / f"{cid}.json", c)
|
|
_write(f, read)
|
|
return {"added": added, "removed": removed, "deleted_cases": deleted, "skipped": skipped}
|
|
|
|
|
|
def check():
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root = Path(d)
|
|
(root / "auto" / "cases").mkdir(parents=True)
|
|
(root / "manual").mkdir()
|
|
a = {"id": "a", "kind": "phenomenon", "name": "A", "family": "Visibility", "signature": "a", "sources": ["S-1"]}
|
|
b = {"id": "b", "kind": "false friend", "name": "B", "family": None, "signature": "b", "sources": ["user"], "review": "2026-11-20"}
|
|
for c in (b, a):
|
|
(root / "auto" / "cases" / f"{c['id']}.json").write_text(json.dumps(c))
|
|
(root / "auto" / "cases" / ".read.json").write_text("{}")
|
|
(root / "manual" / "S.jsonl").write_text(json.dumps({"id": "S-1", "description": "the user's"}) + "\n")
|
|
(root / "auto" / "S.jsonl").write_text(json.dumps({"id": "S-1", "start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "Visibility", "description": "an agent's"}) + "\n")
|
|
assert [c["id"] for c in cases(root)] == ["a", "b"] # sorted, dotfile ignored
|
|
assert labels(root)["S-1"]["description"] == "an agent's" and labels(root)["S-1"]["station"] == "S"
|
|
assert listing(cases(root)) == ["a | phenomenon | Visibility | A", "b | false friend | - | B"]
|
|
assert provenance(b) == "validated by the user on 2026-11-20" and provenance(a) == "not validated"
|
|
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
|
|
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec)] == ["b", "a"]
|
|
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec, family="Visibility")] == ["b", "a"] # null kept
|
|
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec, family="Obstruction")] == ["b"]
|
|
shown = show(0.5, {**a, "sources": ["S-1", "Z-9", "expert"]}, labels(root)).splitlines()
|
|
assert shown[0] == "## a — phenomenon — Visibility — similarity 0.50; not validated"
|
|
assert shown[1] == "A" # the name: a false friend's statement
|
|
assert shown[-1] == "sources: S-1 (S 2026-11-04T00:00Z → 2026-11-04T01:00Z); Z-9 (not in auto/); expert"
|
|
before = globals()["embed"] # --list without Ollama
|
|
globals()["embed"] = lambda *x: (_ for _ in ()).throw(AssertionError("--list called Ollama"))
|
|
try:
|
|
import contextlib, io
|
|
with contextlib.redirect_stdout(io.StringIO()) as out:
|
|
main_list(root)
|
|
assert out.getvalue() == "a | phenomenon | Visibility | A\nb | false friend | - | B\n"
|
|
finally:
|
|
globals()["embed"] = before
|
|
(root / "auto" / "cases" / "c.json").write_text("{broken")
|
|
try:
|
|
cases(root)
|
|
raise AssertionError("a broken case must stop with its path")
|
|
except SystemExit as e:
|
|
assert "c.json" in str(e)
|
|
with tempfile.TemporaryDirectory() as d: # the bookkeeping of a refresh
|
|
root = Path(d); cdir = root / "auto" / "cases"; cdir.mkdir(parents=True)
|
|
|
|
def put_labels(*ls):
|
|
(root / "auto" / "S.jsonl").write_text("".join(json.dumps(l) + "\n" for l in ls))
|
|
|
|
def case(i, sources, **kw):
|
|
(cdir / f"{i}.json").write_text(json.dumps({"id": i, "kind": "phenomenon", "name": i, "family": None, "signature": i, "sources": sources, **kw}))
|
|
|
|
def get(i):
|
|
return json.loads((cdir / f"{i}.json").read_text())
|
|
quiet = __import__("contextlib").redirect_stderr(__import__("io").StringIO())
|
|
lab = {"id": "S-1", "start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "No data", "description": "gap"}
|
|
(root / "auto" / "T.jsonl").write_text(json.dumps({**lab, "id": "T-5", "family": "Visibility", "description": "fog"}) + "\n")
|
|
put_labels(lab); case("c1", ["user"], review="2026-11-20"); case("c2", ["T-5"])
|
|
todo, deleted = pending(root)
|
|
assert sorted(l["id"] for ls in todo.values() for l in ls) == ["S-1", "T-5"] and deleted == []
|
|
r = record({"S-1": ["c1"], "T-5": ["c2"]}, root)
|
|
assert r["added"] == {"c1": ["S-1"]} and get("c1")["sources"] == ["user", "S-1"] # c2 had T-5 already
|
|
assert get("c1")["review"] == "2026-11-20" # a new source keeps the validation
|
|
assert pending(root) == ({}, [])
|
|
put_labels({**lab, "description": "longer gap"}) # same id, text changed: read again
|
|
assert [l["id"] for l in pending(root)[0]["No data"]] == ["S-1"]
|
|
record({"S-1": ["c2"]}, root)
|
|
assert get("c1")["sources"] == ["user"] and "S-1" in get("c2")["sources"]
|
|
record({"S-1": []}, root) # no case: stays to read
|
|
assert "S-1" in [l["id"] for l in pending(root)[0]["No data"]]
|
|
try:
|
|
with quiet:
|
|
record({"S-1": ["nope"]}, root)
|
|
raise AssertionError("unknown case must refuse")
|
|
except SystemExit:
|
|
pass
|
|
record({"S-1": ["c2"]}, root)
|
|
(cdir / "c2.json").unlink() # deleted by hand by the user
|
|
assert "S-1" in [l["id"] for l in pending(root)[0]["No data"]]
|
|
case("c3", ["S-1", "T-5"]); record({"S-1": ["c3"]}, root)
|
|
put_labels() # S-1 deleted by aimsight-app
|
|
assert pending(root)[1] == ["S-1"]
|
|
r = record({"S-1": ["c3"]}, root) # assigned after its deletion
|
|
assert r["skipped"] == ["S-1"] and get("c3")["sources"] == ["T-5"]
|
|
case("c4", ["S-2"]); r = record({}, root) # a seeded source never recorded, gone from auto/
|
|
assert r["deleted_cases"] == ["c4"] and not (cdir / "c4.json").exists()
|
|
assert (cdir / "c1.json").exists() and get("c3")["sources"] == ["T-5"] # non-label source: never deleted
|
|
pending_dir = root / "work"; write_pending({"No data": [lab]}, pending_dir)
|
|
assert (pending_dir / "no-data.jsonl").exists()
|
|
for name in ("S.jsonl", "visibility.answer.json", "assignments.json"):
|
|
(pending_dir / name).write_text("{}")
|
|
write_pending({}, pending_dir) # a wrong DIR must not lose station files
|
|
assert (pending_dir / "S.jsonl").exists()
|
|
assert not any((pending_dir / n).exists() for n in ("no-data.jsonl", "visibility.answer.json", "assignments.json"))
|
|
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
|
|
|
|
class Answer:
|
|
def __init__(self, r):
|
|
self.r = r
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *a):
|
|
return False
|
|
|
|
def read(self):
|
|
return json.dumps(self.r).encode()
|
|
|
|
def fake_urlopen(*a, **k):
|
|
r = next(answers)
|
|
if isinstance(r, Exception):
|
|
raise r
|
|
return Answer(r)
|
|
before = urllib.request.urlopen, globals()["start_ollama"]
|
|
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
|
|
try:
|
|
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
|
|
finally:
|
|
urllib.request.urlopen, globals()["start_ollama"] = before
|
|
print("ok")
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
|
|
p.add_argument("--k", type=int, default=8, help="number of cases (8)")
|
|
p.add_argument("--family", help="only this family (key of families.md)")
|
|
p.add_argument("--list", action="store_true", help="every case, one line each, without Ollama")
|
|
p.add_argument("--index", action="store_true", help="updates the index and stops")
|
|
p.add_argument("--pending", metavar="DIR", help="writes the labels to read, one file per family, in DIR")
|
|
p.add_argument("--record", metavar="FILE", help='applies {"assignments": {label id: [case ids]}} from FILE')
|
|
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
|
|
a = p.parse_args()
|
|
if a.check:
|
|
return check()
|
|
if a.list:
|
|
return main_list()
|
|
if a.pending:
|
|
todo, deleted = pending()
|
|
write_pending(todo, Path(a.pending))
|
|
for family, ls in sorted(todo.items()):
|
|
print(f"{family}: {len(ls)} labels to read")
|
|
return print(f"deleted since the last record: {deleted or 'none'}")
|
|
if a.record:
|
|
return print(json.dumps(record(json.loads(Path(a.record).read_text(encoding="utf-8"))["assignments"]), indent=1))
|
|
everything = cases()
|
|
vectors = updated_index(everything)
|
|
if a.index:
|
|
return print(f"{len(vectors)} cases indexed ({MODEL}): {INDEX}")
|
|
if not a.query:
|
|
p.error("give a text, --list or --index")
|
|
known = labels()
|
|
for score, c in search(embed([a.query])[0], everything, vectors, a.k, a.family):
|
|
print(show(score, c, known))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|