The labeling scripts in scripts/ (validate, ids, precedents); the index stays in .index/

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 13:20:31 +01:00
1 parent 4688d8ab9e
commit f34616ae38
6 files changed
+22 -22

No files matched your search

+64
View File
@@ -0,0 +1,64 @@
"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
python3 labeling/scripts/ids.py # gives an id to the labels that have none, every station of labeling/auto/
python3 labeling/scripts/ids.py NORDEND # one station
A label keeps its id when it is changed; an added label receives the next one of its station.
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
"""
import glob
import json
import os
import sys
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/labeling/auto, or through the labeling/ link
def id_first(e):
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
if "id" in e and next(iter(e)) != "id":
rest = {k: v for k, v in e.items() if k != "id"}
ident = e["id"]
e.clear()
e.update({"id": ident}, **rest)
def assign(station, labels, removed=()):
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
n = max(taken, default=0)
given = 0
for e in labels:
if "id" not in e:
n += 1
e["id"] = f"{station}-{n}"
given += 1
id_first(e)
return given
def update_file(station):
p = os.path.join(AUTO, f"{station}.jsonl")
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
labels = [json.loads(l) for l in lines]
assign(station, labels)
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
tmp = p + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
f.writelines(new)
os.replace(tmp, p)
return labels
if __name__ == "__main__":
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
for st in stations:
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
labels = update_file(st)
ids = [e["id"] for e in labels]
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
print(f"{st}: {len(labels)} labels, {before} ids given")
+196
View File
@@ -0,0 +1,196 @@
"""The precedents: the labels of auto/ closest to a text, by meaning (RAG). manual/ is the user's alone: no process
uses it.
The analyst describes what it sees on its range; the tool returns the labels already written that speak of it most, all
stations, with their provenance (reviewed, not reviewed): it is up to the analyst to judge whether they fit its case.
python3 labeling/scripts/precedents.py "the targets jump together when a reference is rejected intermittently"
python3 labeling/scripts/precedents.py "<text>" --exclude NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --family Obstruction --k 5
python3 labeling/scripts/precedents.py --index # only updates the index
python3 labeling/scripts/precedents.py --check # checks the logic, without Ollama
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed labels go through
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
"""
import argparse
import glob
import hashlib
import json
import math
import os
import subprocess
import sys
import time
import urllib.request
from datetime import datetime
from pathlib import Path
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
INDEX = HERE / ".index" / f"{MODEL.replace(':', '_')}.json"
BATCH = 32 # texts per call to the model
def labels(root=HERE):
"""Every label of auto/, with its station and its provenance; never those of manual/."""
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
station = Path(f).stem
for line in open(f, encoding="utf-8"):
if line.strip():
e = json.loads(line)
e["station"], e["provenance"] = station, provenance(e)
yield e
def provenance(e):
if e.get("review"):
return f"auto, reviewed on {e['review']}"
return "auto, not reviewed" + (f" (analysis {e['analysis']})" if e.get("analysis") else "")
def text(e):
return f"{e.get('family', '')}. {e.get('description', '')}"
def key(t):
return hashlib.sha1(t.encode("utf-8")).hexdigest()
def parse_time(s):
return datetime.fromisoformat(s.replace("Z", "+00:00"))
def overlaps(e, station, start, end):
"""Does the label touch the excluded range (same station, overlapping intervals)?"""
return e["station"] == station and parse_time(e["start"]) <= parse_time(end) and parse_time(e["end"]) >= parse_time(start)
def norm(v):
n = math.sqrt(sum(x * x for x in v)) or 1.0
return [round(x / n, 5) for x in v]
def start_ollama(wait=120):
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
return False
deadline = time.monotonic() + wait
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
return True
except OSError:
time.sleep(2)
return False
def embed(texts):
body = json.dumps({"model": MODEL, "input": texts}).encode()
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
try:
with urllib.request.urlopen(req, timeout=300) as r:
return [norm(v) for v in json.load(r)["embeddings"]]
except OSError as err:
if attempt == 2 or not start_ollama():
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
def updated_index(everything):
"""The vectors of every label; computes the missing ones, forgets those of vanished texts."""
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
wanted = {key(text(e)): text(e) for e in everything}
missing = [k for k in wanted if k not in vectors]
for i in range(0, len(missing), BATCH):
batch = missing[i:i + BATCH]
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
if missing or len(vectors) != len(wanted):
vectors = {k: vectors[k] for k in wanted}
INDEX.parent.mkdir(exist_ok=True)
INDEX.write_text(json.dumps(vectors))
return vectors
def search(query, everything, vectors, q, k=8, family=None, exclude=None):
kept = [e for e in everything if (not family or e.get("family") == family) and not (exclude and overlaps(e, *exclude))]
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(e))])), e) for e in kept]
return sorted(scores, key=lambda n: -n[0])[:k]
def check():
e = {"station": "S", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"}
assert overlaps(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
assert overlaps(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # common edge: excluded
assert not overlaps(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z")
assert not overlaps(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
assert provenance({"review": "2026-11-04"}) == "auto, reviewed on 2026-11-04"
import tempfile # only the labels of auto/ are used: manual/ is the user's alone
with tempfile.TemporaryDirectory() as d:
for sub, desc in (("manual", "the user's"), ("auto", "an agent's")):
(Path(d) / sub).mkdir()
(Path(d) / sub / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8")
assert [x["description"] for x in labels(Path(d))] == ["an agent's"]
assert provenance({"analysis": "X"}) == "auto, not reviewed (analysis X)"
a, b = {**e, "family": "A", "description": "a"}, {**e, "station": "T", "family": "B", "description": "b"}
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0])] == ["B", "A"]
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], exclude=("T", e["start"], e["end"]))] == ["A"]
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], family="A")] == ["A"]
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
class Answer:
def __init__(self, r):
self.r = r
def __enter__(self):
return self
def __exit__(self, *a):
return False
def read(self):
return json.dumps(self.r).encode()
def fake_urlopen(*a, **k):
r = next(answers)
if isinstance(r, Exception):
raise r
return Answer(r)
before = urllib.request.urlopen, globals()["start_ollama"]
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
try:
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
finally:
urllib.request.urlopen, globals()["start_ollama"] = before
print("ok")
def main():
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
p.add_argument("--k", type=int, default=8, help="number of precedents (8)")
p.add_argument("--family", help="only this family (key of families.md)")
p.add_argument("--exclude", nargs=3, metavar=("STATION", "START", "END"),
help="the range you analyse: its labels are left out")
p.add_argument("--index", action="store_true", help="updates the index and stops")
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
a = p.parse_args()
if a.check:
return check()
everything = list(labels())
vectors = updated_index(everything)
if a.index:
return print(f"{len(vectors)} labels indexed ({MODEL}): {INDEX}")
if not a.query:
p.error("give a text, or --index")
for score, e in search(a.query, everything, vectors, embed([a.query])[0], a.k, a.family, a.exclude):
print(f"## {e.get('id', 'no id')} — {e.get('family') or 'no family'} — {e['station']} {e['start']} → {e['end']}\n"
f"similarity {score:.2f}; {e['provenance']}\n{e['description']}\n")
if __name__ == "__main__":
main()
+79
View File
@@ -0,0 +1,79 @@
"""Validation of the automatic labels (labeling/auto/*.jsonl) against the rules of labeling/labeling-rules.md.
python3 labeling/scripts/validate.py # every station
python3 labeling/scripts/validate.py NORDEND # one station
python3 labeling/scripts/validate.py <folder>/labels.jsonl # a proposed file, before it is submitted (no `review` required)
Errors (to fix): unreadable JSON, missing field, unknown family, malformed date, start >= end,
label without `review` or without `id` (`python3 labeling/scripts/ids.py`), id not first in the line, duplicate id, exact duplicate,
numeric value, local time, "probably not a movement". "Nothing to worry about" is right only with an established cause: to
judge by reading, not by script.
Flagged (to look at): overlaps of two labels of the same family on the same station.
"""
import glob
import json
import os
import re
import sys
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/labeling/auto, or through the labeling/ link
FAMILIES = {"Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"}
DATE = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\dZ$")
# A decimal comma (names such as F4.2 and dates such as 05.10 carry a dot), or a number followed by a unit.
NUMBER = re.compile(r"(?<![\w.])\d+,\d+|\b\d+\s*(noise|noises|px|°|%|MAD|mm|times)(?!\w)|\b\d+\s+(targets|measurements) (out )?of\b")
FORBIDDEN = re.compile(r"local time|local morning|local evening|local day|probably not a movement", re.I)
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
errors, flagged, total = [], [], 0
for st in stations:
seen, ids, labels = set(), set(), []
proposed = st.endswith(".jsonl") # a proposed file: its lines have no `review` nor `id` yet
path = st if proposed else os.path.join(AUTO, f"{st}.jsonl")
for n, line in enumerate(open(path, encoding="utf-8"), 1):
if not line.strip():
continue
total += 1
where = f"{st}#{n}"
try:
e = json.loads(line)
except json.JSONDecodeError:
errors.append(f"{where} unreadable JSON"); continue
missing = ({"start", "end", "family", "description"} | (set() if proposed else {"review", "id"})) - e.keys()
if missing:
errors.append(f"{where} missing field: {sorted(missing)}")
if "id" in e:
if next(iter(e)) != "id":
errors.append(f"{where} id not first in the line (`python3 labeling/scripts/ids.py`)")
if e["id"] in ids:
errors.append(f"{where} duplicate id: {e['id']}")
ids.add(e["id"])
if e.get("family") not in FAMILIES:
errors.append(f"{where} unknown family: {e.get('family')}")
if not (DATE.match(e.get("start", "")) and DATE.match(e.get("end", ""))):
errors.append(f"{where} malformed date"); continue
if e["start"] >= e["end"]:
errors.append(f"{where} start >= end ({e['start']})")
key = (e["start"], e["end"], e["family"], e["description"])
if key in seen:
errors.append(f"{where} exact duplicate")
seen.add(key)
d = e.get("description", "")
if m := NUMBER.search(d):
errors.append(f"{where} numeric value: \"{m.group(0)}\"")
if m := FORBIDDEN.search(d):
errors.append(f"{where} forbidden: \"{m.group(0)}\"")
labels.append((where, e))
for i, (a, x) in enumerate(labels):
for b, y in labels[i + 1:]:
if x["family"] == y["family"] and x["start"] < y["end"] and y["start"] < x["end"]:
flagged.append(f"{a} and {b}: two {x['family']} that overlap")
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
for l in errors:
print(" ERROR", l)
for l in flagged[:40]:
print(" to look at", l)
if len(flagged) > 40:
print(f" … and {len(flagged) - 40} other overlaps")
sys.exit(1 if errors else 0)