This commit is contained in:
maxime committed 2026-11-18 10:51:31 +01:00
1 parent 8e4ebca342
commit 58b630f2d5
112 files changed
+3172 -295

No files matched your search

+396
View File
@@ -0,0 +1,396 @@
"""The cases: the phenomena, false friends and lessons drawn from the labels, one JSON file each in auto/cases/ (the
design: docs/superpowers/specs/2026-11-18-cases-design.md); the RAG that finds those closest to a text, and the
bookkeeping of their refresh by the `cases` skill (.claude/skills/cases/). manual/ is the user's alone: no process uses it.
The analyst describes what it sees on its range; the tool returns the cases that speak of it most, each with its
signature, its checks, its neighbours and how to tell them apart, its sources and whether the user validated it: it is
up to the analyst to judge whether they fit.
python3 labeling/scripts/cases.py "the targets jump together when a reference is rejected intermittently"
python3 labeling/scripts/cases.py "<text>" --family Obstruction --k 5
python3 labeling/scripts/cases.py --list # every case, one line each, without Ollama
python3 labeling/scripts/cases.py --index # only updates the index
python3 labeling/scripts/cases.py --check # checks the logic, without Ollama
python3 labeling/scripts/cases.py --pending DIR # the refresh: the labels to read, one file per family, in DIR
python3 labeling/scripts/cases.py --record FILE # the refresh: {"assignments": {label id: [case ids]}} applied
auto/cases/.read.json, the build's state: per label id, the hash of the text it was read with and the cases it went to.
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed cases go through
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
"""
import argparse
import glob
import hashlib
import json
import math
import os
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
INDEX = HERE / ".index" / f"cases_{MODEL.replace(':', '_')}.json"
BATCH = 32 # texts per call to the model
def cases(root=HERE):
"""Every case of auto/cases/ (not the dotfiles, the build's state), sorted by id."""
out = []
for f in sorted(glob.glob(str(root / "auto" / "cases" / "*.json"))):
try:
out.append(json.loads(Path(f).read_text(encoding="utf-8")))
except json.JSONDecodeError:
sys.exit(f"{f}: unreadable JSON")
return sorted(out, key=lambda c: c["id"])
def labels(root=HERE):
"""Every label of auto/ by id, with its station; never those of manual/."""
out = {}
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
for line in open(f, encoding="utf-8"):
if line.strip():
e = json.loads(line)
out[e["id"]] = e | {"station": Path(f).stem}
return out
def provenance(c):
return f"validated by the user on {c['review']}" if c.get("review") else "not validated"
def text(c):
return f"{c['kind']}. {c.get('family') or ''}. {c['name']}. {c['signature']}"
def listing(cs):
return [f"{c['id']} | {c['kind']} | {c.get('family') or '-'} | {c['name']}" for c in cs]
def key(t):
return hashlib.sha1(t.encode("utf-8")).hexdigest()
def norm(v):
n = math.sqrt(sum(x * x for x in v)) or 1.0
return [round(x / n, 5) for x in v]
def start_ollama(wait=120):
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
return False
deadline = time.monotonic() + wait
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
return True
except OSError:
time.sleep(2)
return False
def embed(texts):
body = json.dumps({"model": MODEL, "input": texts}).encode()
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
try:
with urllib.request.urlopen(req, timeout=300) as r:
return [norm(v) for v in json.load(r)["embeddings"]]
except OSError as err:
if attempt == 2 or not start_ollama():
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
def updated_index(everything):
"""The vectors of every case; computes the missing ones, forgets those of vanished texts."""
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
wanted = {key(text(e)): text(e) for e in everything}
missing = [k for k in wanted if k not in vectors]
for i in range(0, len(missing), BATCH):
batch = missing[i:i + BATCH]
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
if missing or len(vectors) != len(wanted):
vectors = {k: vectors[k] for k in wanted}
INDEX.parent.mkdir(exist_ok=True)
INDEX.write_text(json.dumps(vectors))
return vectors
def search(q, everything, vectors, k=8, family=None):
"""The k cases closest to the query vector q (of `family` and the cross-family ones, if given) -> [(similarity, case)]."""
kept = [c for c in everything if not family or c.get("family") in (family, None)]
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(c))])), c) for c in kept]
return sorted(scores, key=lambda n: -n[0])[:k]
def main_list(root=HERE):
for line in listing(cases(root)):
print(line)
def show(score, c, known):
"""One case found, in full, its label sources dated from `known` (the labels of auto/ by id)."""
def source(s):
e = known.get(s)
return f"{s} ({e['station']} {e['start']} → {e['end']})" if e else (s if s in ("user", "expert", "client report") else f"{s} (not in auto/)")
out = [f"## {c['id']} — {c['kind']} — {c.get('family') or 'no family'} — similarity {score:.2f}; {provenance(c)}",
c["name"], c["signature"]]
out += [f"check: {x}" for x in c.get("checks") or []]
out += [f"neighbour {n['case']}: {n['tell_apart']}" for n in c.get("neighbours") or []]
if c.get("client"):
out.append(f"client: {c['client']}")
out.append("sources: " + "; ".join(source(s) for s in c["sources"]))
return "\n".join(out) + "\n"
def label_hash(label):
"""What a label says, for the refresh: a label read again when its bounds, family or description change."""
return key(f"{label['start']} {label['end']} {label['family']}. {label['description']}")
def _read_path(root):
return root / "auto" / "cases" / ".read.json"
def _write(path, obj):
"""Writes a JSON file, readable in a diff, next to it then replaces it: an interruption does not damage it."""
tmp = path.with_name(path.name + ".tmp")
tmp.write_text(json.dumps(obj, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
os.replace(tmp, path)
def pending(root=HERE):
"""The labels to read, by family (new, changed, or attached to no existing case), and the ids deleted since the
last record -> ({family: [label]}, [id])."""
known, existing = labels(root), {c["id"] for c in cases(root)}
f = _read_path(root)
read = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {}
todo = {}
for i, e in known.items():
r = read.get(i)
if not r or r["hash"] != label_hash(e) or not r["cases"] or any(c not in existing for c in r["cases"]):
todo.setdefault(e["family"], []).append(e)
return todo, sorted(i for i in read if i not in known)
def write_pending(todo, out):
"""One file per family in `out`, `<family in lowercase, dashes>.jsonl`, one label per line. Removes first the files
of an earlier refresh it owns (labels, answers, assignments), and only those: a wrong `out` loses nothing else."""
from validate import FAMILIES
out.mkdir(parents=True, exist_ok=True)
for family in FAMILIES:
for suffix in (".jsonl", ".answer.json"):
(out / f"{family.lower().replace(' ', '-')}{suffix}").unlink(missing_ok=True)
(out / "assignments.json").unlink(missing_ok=True)
for family, ls in todo.items():
(out / f"{family.lower().replace(' ', '-')}.jsonl").write_text(
"".join(json.dumps(e, ensure_ascii=False) + "\n" for e in ls), encoding="utf-8")
def record(assignments, root=HERE):
"""Applies {label id: [case ids]} (the cases each label read illustrates) to the sources of the cases and to
.read.json; removes from every case the label sources no longer in auto/; deletes the cases left without a source
(a non-label source keeps a case). Never touches `review` -> {added, removed, deleted_cases, skipped}."""
from validate import NON_LABEL
known = labels(root)
before = {c["id"]: c for c in cases(root)}
cs = json.loads(json.dumps(before))
if unknown := sorted({i for v in assignments.values() for i in v if i not in cs}):
sys.exit(f"unknown case(s): {unknown}; nothing written")
f = _read_path(root)
read = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {}
added, removed = {}, {}
skipped = sorted(i for i in assignments if i not in known)
for c in cs.values():
for s in [s for s in c["sources"] if s not in NON_LABEL and s not in known]:
c["sources"].remove(s)
removed.setdefault(c["id"], []).append(s)
read = {i: r for i, r in read.items() if i in known}
for i, new in assignments.items():
if i in skipped:
continue
for c in cs.values():
if i in c["sources"] and c["id"] not in new:
c["sources"].remove(i)
removed.setdefault(c["id"], []).append(i)
for cid in new:
if i not in cs[cid]["sources"]:
cs[cid]["sources"].append(i)
added.setdefault(cid, []).append(i)
read[i] = {"hash": label_hash(known[i]), "cases": list(new)}
deleted = sorted(cid for cid, c in cs.items() if not c["sources"])
folder = root / "auto" / "cases"
for cid, c in cs.items():
if cid in deleted:
(folder / f"{cid}.json").unlink()
elif c != before[cid]:
_write(folder / f"{cid}.json", c)
_write(f, read)
return {"added": added, "removed": removed, "deleted_cases": deleted, "skipped": skipped}
def check():
import tempfile
with tempfile.TemporaryDirectory() as d:
root = Path(d)
(root / "auto" / "cases").mkdir(parents=True)
(root / "manual").mkdir()
a = {"id": "a", "kind": "phenomenon", "name": "A", "family": "Visibility", "signature": "a", "sources": ["S-1"]}
b = {"id": "b", "kind": "false friend", "name": "B", "family": None, "signature": "b", "sources": ["user"], "review": "2026-11-20"}
for c in (b, a):
(root / "auto" / "cases" / f"{c['id']}.json").write_text(json.dumps(c))
(root / "auto" / "cases" / ".read.json").write_text("{}")
(root / "manual" / "S.jsonl").write_text(json.dumps({"id": "S-1", "description": "the user's"}) + "\n")
(root / "auto" / "S.jsonl").write_text(json.dumps({"id": "S-1", "start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "Visibility", "description": "an agent's"}) + "\n")
assert [c["id"] for c in cases(root)] == ["a", "b"] # sorted, dotfile ignored
assert labels(root)["S-1"]["description"] == "an agent's" and labels(root)["S-1"]["station"] == "S"
assert listing(cases(root)) == ["a | phenomenon | Visibility | A", "b | false friend | - | B"]
assert provenance(b) == "validated by the user on 2026-11-20" and provenance(a) == "not validated"
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec)] == ["b", "a"]
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec, family="Visibility")] == ["b", "a"] # null kept
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec, family="Obstruction")] == ["b"]
shown = show(0.5, {**a, "sources": ["S-1", "Z-9", "expert"]}, labels(root)).splitlines()
assert shown[0] == "## a — phenomenon — Visibility — similarity 0.50; not validated"
assert shown[1] == "A" # the name: a false friend's statement
assert shown[-1] == "sources: S-1 (S 2026-11-04T00:00Z → 2026-11-04T01:00Z); Z-9 (not in auto/); expert"
before = globals()["embed"] # --list without Ollama
globals()["embed"] = lambda *x: (_ for _ in ()).throw(AssertionError("--list called Ollama"))
try:
import contextlib, io
with contextlib.redirect_stdout(io.StringIO()) as out:
main_list(root)
assert out.getvalue() == "a | phenomenon | Visibility | A\nb | false friend | - | B\n"
finally:
globals()["embed"] = before
(root / "auto" / "cases" / "c.json").write_text("{broken")
try:
cases(root)
raise AssertionError("a broken case must stop with its path")
except SystemExit as e:
assert "c.json" in str(e)
with tempfile.TemporaryDirectory() as d: # the bookkeeping of a refresh
root = Path(d); cdir = root / "auto" / "cases"; cdir.mkdir(parents=True)
def put_labels(*ls):
(root / "auto" / "S.jsonl").write_text("".join(json.dumps(l) + "\n" for l in ls))
def case(i, sources, **kw):
(cdir / f"{i}.json").write_text(json.dumps({"id": i, "kind": "phenomenon", "name": i, "family": None, "signature": i, "sources": sources, **kw}))
def get(i):
return json.loads((cdir / f"{i}.json").read_text())
quiet = __import__("contextlib").redirect_stderr(__import__("io").StringIO())
lab = {"id": "S-1", "start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "No data", "description": "gap"}
(root / "auto" / "T.jsonl").write_text(json.dumps({**lab, "id": "T-5", "family": "Visibility", "description": "fog"}) + "\n")
put_labels(lab); case("c1", ["user"], review="2026-11-20"); case("c2", ["T-5"])
todo, deleted = pending(root)
assert sorted(l["id"] for ls in todo.values() for l in ls) == ["S-1", "T-5"] and deleted == []
r = record({"S-1": ["c1"], "T-5": ["c2"]}, root)
assert r["added"] == {"c1": ["S-1"]} and get("c1")["sources"] == ["user", "S-1"] # c2 had T-5 already
assert get("c1")["review"] == "2026-11-20" # a new source keeps the validation
assert pending(root) == ({}, [])
put_labels({**lab, "description": "longer gap"}) # same id, text changed: read again
assert [l["id"] for l in pending(root)[0]["No data"]] == ["S-1"]
record({"S-1": ["c2"]}, root)
assert get("c1")["sources"] == ["user"] and "S-1" in get("c2")["sources"]
record({"S-1": []}, root) # no case: stays to read
assert "S-1" in [l["id"] for l in pending(root)[0]["No data"]]
try:
with quiet:
record({"S-1": ["nope"]}, root)
raise AssertionError("unknown case must refuse")
except SystemExit:
pass
record({"S-1": ["c2"]}, root)
(cdir / "c2.json").unlink() # deleted by hand by the user
assert "S-1" in [l["id"] for l in pending(root)[0]["No data"]]
case("c3", ["S-1", "T-5"]); record({"S-1": ["c3"]}, root)
put_labels() # S-1 deleted by aimsight-app
assert pending(root)[1] == ["S-1"]
r = record({"S-1": ["c3"]}, root) # assigned after its deletion
assert r["skipped"] == ["S-1"] and get("c3")["sources"] == ["T-5"]
case("c4", ["S-2"]); r = record({}, root) # a seeded source never recorded, gone from auto/
assert r["deleted_cases"] == ["c4"] and not (cdir / "c4.json").exists()
assert (cdir / "c1.json").exists() and get("c3")["sources"] == ["T-5"] # non-label source: never deleted
pending_dir = root / "work"; write_pending({"No data": [lab]}, pending_dir)
assert (pending_dir / "no-data.jsonl").exists()
for name in ("S.jsonl", "visibility.answer.json", "assignments.json"):
(pending_dir / name).write_text("{}")
write_pending({}, pending_dir) # a wrong DIR must not lose station files
assert (pending_dir / "S.jsonl").exists()
assert not any((pending_dir / n).exists() for n in ("no-data.jsonl", "visibility.answer.json", "assignments.json"))
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
class Answer:
def __init__(self, r):
self.r = r
def __enter__(self):
return self
def __exit__(self, *a):
return False
def read(self):
return json.dumps(self.r).encode()
def fake_urlopen(*a, **k):
r = next(answers)
if isinstance(r, Exception):
raise r
return Answer(r)
before = urllib.request.urlopen, globals()["start_ollama"]
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
try:
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
finally:
urllib.request.urlopen, globals()["start_ollama"] = before
print("ok")
def main():
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
p.add_argument("--k", type=int, default=8, help="number of cases (8)")
p.add_argument("--family", help="only this family (key of families.md)")
p.add_argument("--list", action="store_true", help="every case, one line each, without Ollama")
p.add_argument("--index", action="store_true", help="updates the index and stops")
p.add_argument("--pending", metavar="DIR", help="writes the labels to read, one file per family, in DIR")
p.add_argument("--record", metavar="FILE", help='applies {"assignments": {label id: [case ids]}} from FILE')
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
a = p.parse_args()
if a.check:
return check()
if a.list:
return main_list()
if a.pending:
todo, deleted = pending()
write_pending(todo, Path(a.pending))
for family, ls in sorted(todo.items()):
print(f"{family}: {len(ls)} labels to read")
return print(f"deleted since the last record: {deleted or 'none'}")
if a.record:
return print(json.dumps(record(json.loads(Path(a.record).read_text(encoding="utf-8"))["assignments"]), indent=1))
everything = cases()
vectors = updated_index(everything)
if a.index:
return print(f"{len(vectors)} cases indexed ({MODEL}): {INDEX}")
if not a.query:
p.error("give a text, --list or --index")
known = labels()
for score, c in search(embed([a.query])[0], everything, vectors, a.k, a.family):
print(show(score, c, known))
if __name__ == "__main__":
main()