Quality over quantity: only the analyses discussed and validated in the app are marked. validate.py no longer requires review; precedents.py gives the provenance as validated by the user or not. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
197 lines
8.7 KiB
Python
197 lines
8.7 KiB
Python
"""The precedents: the labels of auto/ closest to a text, by meaning (RAG). manual/ is the user's alone: no process
|
|
uses it.
|
|
|
|
The analyst describes what it sees on its range; the tool returns the labels already written that speak of it most, all
|
|
stations, with their provenance (validated by the user, not validated): it is up to the analyst to judge whether they fit its case.
|
|
|
|
python3 labeling/scripts/precedents.py "the targets jump together when a reference is rejected intermittently"
|
|
python3 labeling/scripts/precedents.py "<text>" --exclude NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --family Obstruction --k 5
|
|
python3 labeling/scripts/precedents.py --index # only updates the index
|
|
python3 labeling/scripts/precedents.py --check # checks the logic, without Ollama
|
|
|
|
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed labels go through
|
|
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
|
|
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
|
|
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
|
|
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
|
|
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
|
|
INDEX = HERE / ".index" / f"{MODEL.replace(':', '_')}.json"
|
|
BATCH = 32 # texts per call to the model
|
|
|
|
|
|
def labels(root=HERE):
|
|
"""Every label of auto/, with its station and its provenance; never those of manual/."""
|
|
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
|
|
station = Path(f).stem
|
|
for line in open(f, encoding="utf-8"):
|
|
if line.strip():
|
|
e = json.loads(line)
|
|
e["station"], e["provenance"] = station, provenance(e)
|
|
yield e
|
|
|
|
|
|
def provenance(e):
|
|
if e.get("review"):
|
|
return f"auto, validated by the user on {e['review']}"
|
|
return "auto, not validated" + (f" (analysis {e['analysis']})" if e.get("analysis") else "")
|
|
|
|
|
|
def text(e):
|
|
return f"{e.get('family', '')}. {e.get('description', '')}"
|
|
|
|
|
|
def key(t):
|
|
return hashlib.sha1(t.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def parse_time(s):
|
|
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
|
|
|
|
|
def overlaps(e, station, start, end):
|
|
"""Does the label touch the excluded range (same station, overlapping intervals)?"""
|
|
return e["station"] == station and parse_time(e["start"]) <= parse_time(end) and parse_time(e["end"]) >= parse_time(start)
|
|
|
|
|
|
def norm(v):
|
|
n = math.sqrt(sum(x * x for x in v)) or 1.0
|
|
return [round(x / n, 5) for x in v]
|
|
|
|
|
|
def start_ollama(wait=120):
|
|
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
|
|
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
|
|
return False
|
|
deadline = time.monotonic() + wait
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
|
|
return True
|
|
except OSError:
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
def embed(texts):
|
|
body = json.dumps({"model": MODEL, "input": texts}).encode()
|
|
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
|
|
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=300) as r:
|
|
return [norm(v) for v in json.load(r)["embeddings"]]
|
|
except OSError as err:
|
|
if attempt == 2 or not start_ollama():
|
|
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
|
|
|
|
|
|
def updated_index(everything):
|
|
"""The vectors of every label; computes the missing ones, forgets those of vanished texts."""
|
|
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
|
|
wanted = {key(text(e)): text(e) for e in everything}
|
|
missing = [k for k in wanted if k not in vectors]
|
|
for i in range(0, len(missing), BATCH):
|
|
batch = missing[i:i + BATCH]
|
|
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
|
|
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
|
|
if missing or len(vectors) != len(wanted):
|
|
vectors = {k: vectors[k] for k in wanted}
|
|
INDEX.parent.mkdir(exist_ok=True)
|
|
INDEX.write_text(json.dumps(vectors))
|
|
return vectors
|
|
|
|
|
|
def search(query, everything, vectors, q, k=8, family=None, exclude=None):
|
|
kept = [e for e in everything if (not family or e.get("family") == family) and not (exclude and overlaps(e, *exclude))]
|
|
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(e))])), e) for e in kept]
|
|
return sorted(scores, key=lambda n: -n[0])[:k]
|
|
|
|
|
|
def check():
|
|
e = {"station": "S", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"}
|
|
assert overlaps(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
|
|
assert overlaps(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # common edge: excluded
|
|
assert not overlaps(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z")
|
|
assert not overlaps(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
|
|
assert provenance({"review": "2026-11-04"}) == "auto, validated by the user on 2026-11-04"
|
|
import tempfile # only the labels of auto/ are used: manual/ is the user's alone
|
|
with tempfile.TemporaryDirectory() as d:
|
|
for sub, desc in (("manual", "the user's"), ("auto", "an agent's")):
|
|
(Path(d) / sub).mkdir()
|
|
(Path(d) / sub / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8")
|
|
assert [x["description"] for x in labels(Path(d))] == ["an agent's"]
|
|
assert provenance({"analysis": "X"}) == "auto, not validated (analysis X)"
|
|
a, b = {**e, "family": "A", "description": "a"}, {**e, "station": "T", "family": "B", "description": "b"}
|
|
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
|
|
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0])] == ["B", "A"]
|
|
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], exclude=("T", e["start"], e["end"]))] == ["A"]
|
|
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], family="A")] == ["A"]
|
|
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
|
|
|
|
class Answer:
|
|
def __init__(self, r):
|
|
self.r = r
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *a):
|
|
return False
|
|
|
|
def read(self):
|
|
return json.dumps(self.r).encode()
|
|
|
|
def fake_urlopen(*a, **k):
|
|
r = next(answers)
|
|
if isinstance(r, Exception):
|
|
raise r
|
|
return Answer(r)
|
|
before = urllib.request.urlopen, globals()["start_ollama"]
|
|
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
|
|
try:
|
|
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
|
|
finally:
|
|
urllib.request.urlopen, globals()["start_ollama"] = before
|
|
print("ok")
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
|
|
p.add_argument("--k", type=int, default=8, help="number of precedents (8)")
|
|
p.add_argument("--family", help="only this family (key of families.md)")
|
|
p.add_argument("--exclude", nargs=3, metavar=("STATION", "START", "END"),
|
|
help="the range you analyse: its labels are left out")
|
|
p.add_argument("--index", action="store_true", help="updates the index and stops")
|
|
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
|
|
a = p.parse_args()
|
|
if a.check:
|
|
return check()
|
|
everything = list(labels())
|
|
vectors = updated_index(everything)
|
|
if a.index:
|
|
return print(f"{len(vectors)} labels indexed ({MODEL}): {INDEX}")
|
|
if not a.query:
|
|
p.error("give a text, or --index")
|
|
for score, e in search(a.query, everything, vectors, embed([a.query])[0], a.k, a.family, a.exclude):
|
|
print(f"## {e.get('id', 'no id')} — {e.get('family') or 'no family'} — {e['station']} {e['start']} → {e['end']}\n"
|
|
f"similarity {score:.2f}; {e['provenance']}\n{e['description']}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|