"""The precedents: the labels of auto/ closest to a text, by meaning (RAG). manual/ is the user's alone: no process uses it. The analyst describes what it sees on its range; the tool returns the labels already written that speak of it most, all stations, with their provenance (reviewed, not reviewed): it is up to the analyst to judge whether they fit its case. python3 labeling/scripts/precedents.py "the targets jump together when a reference is rejected intermittently" python3 labeling/scripts/precedents.py "" --exclude NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --family Obstruction --k 5 python3 labeling/scripts/precedents.py --index # only updates the index python3 labeling/scripts/precedents.py --check # checks the logic, without Ollama The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed labels go through the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only. """ import argparse import glob import hashlib import json import math import os import subprocess import sys import time import urllib.request from datetime import datetime from pathlib import Path HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/ URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m") CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer INDEX = HERE / ".index" / f"{MODEL.replace(':', '_')}.json" BATCH = 32 # texts per call to the model def labels(root=HERE): """Every label of auto/, with its station and its provenance; never those of manual/.""" for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))): station = Path(f).stem for line in open(f, encoding="utf-8"): if line.strip(): e = json.loads(line) e["station"], e["provenance"] = station, provenance(e) yield e def provenance(e): if e.get("review"): return f"auto, reviewed on {e['review']}" return "auto, not reviewed" + (f" (analysis {e['analysis']})" if e.get("analysis") else "") def text(e): return f"{e.get('family', '')}. {e.get('description', '')}" def key(t): return hashlib.sha1(t.encode("utf-8")).hexdigest() def parse_time(s): return datetime.fromisoformat(s.replace("Z", "+00:00")) def overlaps(e, station, start, end): """Does the label touch the excluded range (same station, overlapping intervals)?""" return e["station"] == station and parse_time(e["start"]) <= parse_time(end) and parse_time(e["end"]) >= parse_time(start) def norm(v): n = math.sqrt(sum(x * x for x in v)) or 1.0 return [round(x / n, 5) for x in v] def start_ollama(wait=120): """The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers.""" if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode: return False deadline = time.monotonic() + wait while time.monotonic() < deadline: try: with urllib.request.urlopen(f"{URL}/api/version", timeout=5): return True except OSError: time.sleep(2) return False def embed(texts): body = json.dumps({"model": MODEL, "input": texts}).encode() req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"}) for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried try: with urllib.request.urlopen(req, timeout=300) as r: return [norm(v) for v in json.load(r)["embeddings"]] except OSError as err: if attempt == 2 or not start_ollama(): sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.") def updated_index(everything): """The vectors of every label; computes the missing ones, forgets those of vanished texts.""" vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {} wanted = {key(text(e)): text(e) for e in everything} missing = [k for k in wanted if k not in vectors] for i in range(0, len(missing), BATCH): batch = missing[i:i + BATCH] vectors.update(zip(batch, embed([wanted[k] for k in batch]))) print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr) if missing or len(vectors) != len(wanted): vectors = {k: vectors[k] for k in wanted} INDEX.parent.mkdir(exist_ok=True) INDEX.write_text(json.dumps(vectors)) return vectors def search(query, everything, vectors, q, k=8, family=None, exclude=None): kept = [e for e in everything if (not family or e.get("family") == family) and not (exclude and overlaps(e, *exclude))] scores = [(sum(a * b for a, b in zip(q, vectors[key(text(e))])), e) for e in kept] return sorted(scores, key=lambda n: -n[0])[:k] def check(): e = {"station": "S", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"} assert overlaps(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z") assert overlaps(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # common edge: excluded assert not overlaps(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z") assert not overlaps(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z") assert provenance({"review": "2026-11-04"}) == "auto, reviewed on 2026-11-04" import tempfile # only the labels of auto/ are used: manual/ is the user's alone with tempfile.TemporaryDirectory() as d: for sub, desc in (("manual", "the user's"), ("auto", "an agent's")): (Path(d) / sub).mkdir() (Path(d) / sub / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8") assert [x["description"] for x in labels(Path(d))] == ["an agent's"] assert provenance({"analysis": "X"}) == "auto, not reviewed (analysis X)" a, b = {**e, "family": "A", "description": "a"}, {**e, "station": "T", "family": "B", "description": "b"} vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]} assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0])] == ["B", "A"] assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], exclude=("T", e["start"], e["end"]))] == ["A"] assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], family="A")] == ["A"] calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes class Answer: def __init__(self, r): self.r = r def __enter__(self): return self def __exit__(self, *a): return False def read(self): return json.dumps(self.r).encode() def fake_urlopen(*a, **k): r = next(answers) if isinstance(r, Exception): raise r return Answer(r) before = urllib.request.urlopen, globals()["start_ollama"] urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True try: assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls finally: urllib.request.urlopen, globals()["start_ollama"] = before print("ok") def main(): p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) p.add_argument("query", nargs="?", help="what you see, in English, as a label description") p.add_argument("--k", type=int, default=8, help="number of precedents (8)") p.add_argument("--family", help="only this family (key of families.md)") p.add_argument("--exclude", nargs=3, metavar=("STATION", "START", "END"), help="the range you analyse: its labels are left out") p.add_argument("--index", action="store_true", help="updates the index and stops") p.add_argument("--check", action="store_true", help="checks the logic, without Ollama") a = p.parse_args() if a.check: return check() everything = list(labels()) vectors = updated_index(everything) if a.index: return print(f"{len(vectors)} labels indexed ({MODEL}): {INDEX}") if not a.query: p.error("give a text, or --index") for score, e in search(a.query, everything, vectors, embed([a.query])[0], a.k, a.family, a.exclude): print(f"## {e.get('id', 'no id')} — {e.get('family') or 'no family'} — {e['station']} {e['start']} → {e['end']}\n" f"similarity {score:.2f}; {e['provenance']}\n{e['description']}\n") if __name__ == "__main__": main()