The labeling scripts in scripts/ (validate, ids, precedents); the index stays in .index/
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4688d8ab9e
commit
f34616ae38
6 files changed
+22
-22
No files matched your search
@@ -0,0 +1,196 @@
|
||||
"""The precedents: the labels of auto/ closest to a text, by meaning (RAG). manual/ is the user's alone: no process
|
||||
uses it.
|
||||
|
||||
The analyst describes what it sees on its range; the tool returns the labels already written that speak of it most, all
|
||||
stations, with their provenance (reviewed, not reviewed): it is up to the analyst to judge whether they fit its case.
|
||||
|
||||
python3 labeling/scripts/precedents.py "the targets jump together when a reference is rejected intermittently"
|
||||
python3 labeling/scripts/precedents.py "<text>" --exclude NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --family Obstruction --k 5
|
||||
python3 labeling/scripts/precedents.py --index # only updates the index
|
||||
python3 labeling/scripts/precedents.py --check # checks the logic, without Ollama
|
||||
|
||||
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed labels go through
|
||||
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
|
||||
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
|
||||
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
|
||||
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
|
||||
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
|
||||
INDEX = HERE / ".index" / f"{MODEL.replace(':', '_')}.json"
|
||||
BATCH = 32 # texts per call to the model
|
||||
|
||||
|
||||
def labels(root=HERE):
|
||||
"""Every label of auto/, with its station and its provenance; never those of manual/."""
|
||||
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
|
||||
station = Path(f).stem
|
||||
for line in open(f, encoding="utf-8"):
|
||||
if line.strip():
|
||||
e = json.loads(line)
|
||||
e["station"], e["provenance"] = station, provenance(e)
|
||||
yield e
|
||||
|
||||
|
||||
def provenance(e):
|
||||
if e.get("review"):
|
||||
return f"auto, reviewed on {e['review']}"
|
||||
return "auto, not reviewed" + (f" (analysis {e['analysis']})" if e.get("analysis") else "")
|
||||
|
||||
|
||||
def text(e):
|
||||
return f"{e.get('family', '')}. {e.get('description', '')}"
|
||||
|
||||
|
||||
def key(t):
|
||||
return hashlib.sha1(t.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def parse_time(s):
|
||||
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def overlaps(e, station, start, end):
|
||||
"""Does the label touch the excluded range (same station, overlapping intervals)?"""
|
||||
return e["station"] == station and parse_time(e["start"]) <= parse_time(end) and parse_time(e["end"]) >= parse_time(start)
|
||||
|
||||
|
||||
def norm(v):
|
||||
n = math.sqrt(sum(x * x for x in v)) or 1.0
|
||||
return [round(x / n, 5) for x in v]
|
||||
|
||||
|
||||
def start_ollama(wait=120):
|
||||
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
|
||||
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
|
||||
return False
|
||||
deadline = time.monotonic() + wait
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
|
||||
return True
|
||||
except OSError:
|
||||
time.sleep(2)
|
||||
return False
|
||||
|
||||
|
||||
def embed(texts):
|
||||
body = json.dumps({"model": MODEL, "input": texts}).encode()
|
||||
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
|
||||
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=300) as r:
|
||||
return [norm(v) for v in json.load(r)["embeddings"]]
|
||||
except OSError as err:
|
||||
if attempt == 2 or not start_ollama():
|
||||
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
|
||||
|
||||
|
||||
def updated_index(everything):
|
||||
"""The vectors of every label; computes the missing ones, forgets those of vanished texts."""
|
||||
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
|
||||
wanted = {key(text(e)): text(e) for e in everything}
|
||||
missing = [k for k in wanted if k not in vectors]
|
||||
for i in range(0, len(missing), BATCH):
|
||||
batch = missing[i:i + BATCH]
|
||||
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
|
||||
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
|
||||
if missing or len(vectors) != len(wanted):
|
||||
vectors = {k: vectors[k] for k in wanted}
|
||||
INDEX.parent.mkdir(exist_ok=True)
|
||||
INDEX.write_text(json.dumps(vectors))
|
||||
return vectors
|
||||
|
||||
|
||||
def search(query, everything, vectors, q, k=8, family=None, exclude=None):
|
||||
kept = [e for e in everything if (not family or e.get("family") == family) and not (exclude and overlaps(e, *exclude))]
|
||||
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(e))])), e) for e in kept]
|
||||
return sorted(scores, key=lambda n: -n[0])[:k]
|
||||
|
||||
|
||||
def check():
|
||||
e = {"station": "S", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"}
|
||||
assert overlaps(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
|
||||
assert overlaps(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # common edge: excluded
|
||||
assert not overlaps(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z")
|
||||
assert not overlaps(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
|
||||
assert provenance({"review": "2026-11-04"}) == "auto, reviewed on 2026-11-04"
|
||||
import tempfile # only the labels of auto/ are used: manual/ is the user's alone
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
for sub, desc in (("manual", "the user's"), ("auto", "an agent's")):
|
||||
(Path(d) / sub).mkdir()
|
||||
(Path(d) / sub / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8")
|
||||
assert [x["description"] for x in labels(Path(d))] == ["an agent's"]
|
||||
assert provenance({"analysis": "X"}) == "auto, not reviewed (analysis X)"
|
||||
a, b = {**e, "family": "A", "description": "a"}, {**e, "station": "T", "family": "B", "description": "b"}
|
||||
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
|
||||
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0])] == ["B", "A"]
|
||||
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], exclude=("T", e["start"], e["end"]))] == ["A"]
|
||||
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], family="A")] == ["A"]
|
||||
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
|
||||
|
||||
class Answer:
|
||||
def __init__(self, r):
|
||||
self.r = r
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps(self.r).encode()
|
||||
|
||||
def fake_urlopen(*a, **k):
|
||||
r = next(answers)
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return Answer(r)
|
||||
before = urllib.request.urlopen, globals()["start_ollama"]
|
||||
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
|
||||
try:
|
||||
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
|
||||
finally:
|
||||
urllib.request.urlopen, globals()["start_ollama"] = before
|
||||
print("ok")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
|
||||
p.add_argument("--k", type=int, default=8, help="number of precedents (8)")
|
||||
p.add_argument("--family", help="only this family (key of families.md)")
|
||||
p.add_argument("--exclude", nargs=3, metavar=("STATION", "START", "END"),
|
||||
help="the range you analyse: its labels are left out")
|
||||
p.add_argument("--index", action="store_true", help="updates the index and stops")
|
||||
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
|
||||
a = p.parse_args()
|
||||
if a.check:
|
||||
return check()
|
||||
everything = list(labels())
|
||||
vectors = updated_index(everything)
|
||||
if a.index:
|
||||
return print(f"{len(vectors)} labels indexed ({MODEL}): {INDEX}")
|
||||
if not a.query:
|
||||
p.error("give a text, or --index")
|
||||
for score, e in search(a.query, everything, vectors, embed([a.query])[0], a.k, a.family, a.exclude):
|
||||
print(f"## {e.get('id', 'no id')} — {e.get('family') or 'no family'} — {e['station']} {e['start']} → {e['end']}\n"
|
||||
f"similarity {score:.2f}; {e['provenance']}\n{e['description']}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user