The labeling scripts in scripts/ (validate, ids, precedents); the index stays in .index/
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4688d8ab9e
commit
f34616ae38
6 files changed
+22
-22
No files matched your search
@@ -0,0 +1,64 @@
|
||||
"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
|
||||
|
||||
python3 labeling/scripts/ids.py # gives an id to the labels that have none, every station of labeling/auto/
|
||||
python3 labeling/scripts/ids.py NORDEND # one station
|
||||
|
||||
A label keeps its id when it is changed; an added label receives the next one of its station.
|
||||
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/labeling/auto, or through the labeling/ link
|
||||
|
||||
|
||||
def id_first(e):
|
||||
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
|
||||
if "id" in e and next(iter(e)) != "id":
|
||||
rest = {k: v for k, v in e.items() if k != "id"}
|
||||
ident = e["id"]
|
||||
e.clear()
|
||||
e.update({"id": ident}, **rest)
|
||||
|
||||
|
||||
def assign(station, labels, removed=()):
|
||||
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
|
||||
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
|
||||
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
|
||||
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
|
||||
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
|
||||
n = max(taken, default=0)
|
||||
given = 0
|
||||
for e in labels:
|
||||
if "id" not in e:
|
||||
n += 1
|
||||
e["id"] = f"{station}-{n}"
|
||||
given += 1
|
||||
id_first(e)
|
||||
return given
|
||||
|
||||
|
||||
def update_file(station):
|
||||
p = os.path.join(AUTO, f"{station}.jsonl")
|
||||
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
|
||||
labels = [json.loads(l) for l in lines]
|
||||
assign(station, labels)
|
||||
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
|
||||
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
|
||||
tmp = p + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.writelines(new)
|
||||
os.replace(tmp, p)
|
||||
return labels
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
|
||||
for st in stations:
|
||||
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
|
||||
labels = update_file(st)
|
||||
ids = [e["id"] for e in labels]
|
||||
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
|
||||
print(f"{st}: {len(labels)} labels, {before} ids given")
|
||||
@@ -0,0 +1,196 @@
|
||||
"""The precedents: the labels of auto/ closest to a text, by meaning (RAG). manual/ is the user's alone: no process
|
||||
uses it.
|
||||
|
||||
The analyst describes what it sees on its range; the tool returns the labels already written that speak of it most, all
|
||||
stations, with their provenance (reviewed, not reviewed): it is up to the analyst to judge whether they fit its case.
|
||||
|
||||
python3 labeling/scripts/precedents.py "the targets jump together when a reference is rejected intermittently"
|
||||
python3 labeling/scripts/precedents.py "<text>" --exclude NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --family Obstruction --k 5
|
||||
python3 labeling/scripts/precedents.py --index # only updates the index
|
||||
python3 labeling/scripts/precedents.py --check # checks the logic, without Ollama
|
||||
|
||||
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed labels go through
|
||||
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
|
||||
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
|
||||
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
|
||||
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
|
||||
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
|
||||
INDEX = HERE / ".index" / f"{MODEL.replace(':', '_')}.json"
|
||||
BATCH = 32 # texts per call to the model
|
||||
|
||||
|
||||
def labels(root=HERE):
|
||||
"""Every label of auto/, with its station and its provenance; never those of manual/."""
|
||||
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
|
||||
station = Path(f).stem
|
||||
for line in open(f, encoding="utf-8"):
|
||||
if line.strip():
|
||||
e = json.loads(line)
|
||||
e["station"], e["provenance"] = station, provenance(e)
|
||||
yield e
|
||||
|
||||
|
||||
def provenance(e):
|
||||
if e.get("review"):
|
||||
return f"auto, reviewed on {e['review']}"
|
||||
return "auto, not reviewed" + (f" (analysis {e['analysis']})" if e.get("analysis") else "")
|
||||
|
||||
|
||||
def text(e):
|
||||
return f"{e.get('family', '')}. {e.get('description', '')}"
|
||||
|
||||
|
||||
def key(t):
|
||||
return hashlib.sha1(t.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def parse_time(s):
|
||||
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def overlaps(e, station, start, end):
|
||||
"""Does the label touch the excluded range (same station, overlapping intervals)?"""
|
||||
return e["station"] == station and parse_time(e["start"]) <= parse_time(end) and parse_time(e["end"]) >= parse_time(start)
|
||||
|
||||
|
||||
def norm(v):
|
||||
n = math.sqrt(sum(x * x for x in v)) or 1.0
|
||||
return [round(x / n, 5) for x in v]
|
||||
|
||||
|
||||
def start_ollama(wait=120):
|
||||
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
|
||||
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
|
||||
return False
|
||||
deadline = time.monotonic() + wait
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
|
||||
return True
|
||||
except OSError:
|
||||
time.sleep(2)
|
||||
return False
|
||||
|
||||
|
||||
def embed(texts):
|
||||
body = json.dumps({"model": MODEL, "input": texts}).encode()
|
||||
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
|
||||
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=300) as r:
|
||||
return [norm(v) for v in json.load(r)["embeddings"]]
|
||||
except OSError as err:
|
||||
if attempt == 2 or not start_ollama():
|
||||
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
|
||||
|
||||
|
||||
def updated_index(everything):
|
||||
"""The vectors of every label; computes the missing ones, forgets those of vanished texts."""
|
||||
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
|
||||
wanted = {key(text(e)): text(e) for e in everything}
|
||||
missing = [k for k in wanted if k not in vectors]
|
||||
for i in range(0, len(missing), BATCH):
|
||||
batch = missing[i:i + BATCH]
|
||||
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
|
||||
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
|
||||
if missing or len(vectors) != len(wanted):
|
||||
vectors = {k: vectors[k] for k in wanted}
|
||||
INDEX.parent.mkdir(exist_ok=True)
|
||||
INDEX.write_text(json.dumps(vectors))
|
||||
return vectors
|
||||
|
||||
|
||||
def search(query, everything, vectors, q, k=8, family=None, exclude=None):
|
||||
kept = [e for e in everything if (not family or e.get("family") == family) and not (exclude and overlaps(e, *exclude))]
|
||||
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(e))])), e) for e in kept]
|
||||
return sorted(scores, key=lambda n: -n[0])[:k]
|
||||
|
||||
|
||||
def check():
|
||||
e = {"station": "S", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"}
|
||||
assert overlaps(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
|
||||
assert overlaps(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # common edge: excluded
|
||||
assert not overlaps(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z")
|
||||
assert not overlaps(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
|
||||
assert provenance({"review": "2026-11-04"}) == "auto, reviewed on 2026-11-04"
|
||||
import tempfile # only the labels of auto/ are used: manual/ is the user's alone
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
for sub, desc in (("manual", "the user's"), ("auto", "an agent's")):
|
||||
(Path(d) / sub).mkdir()
|
||||
(Path(d) / sub / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8")
|
||||
assert [x["description"] for x in labels(Path(d))] == ["an agent's"]
|
||||
assert provenance({"analysis": "X"}) == "auto, not reviewed (analysis X)"
|
||||
a, b = {**e, "family": "A", "description": "a"}, {**e, "station": "T", "family": "B", "description": "b"}
|
||||
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
|
||||
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0])] == ["B", "A"]
|
||||
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], exclude=("T", e["start"], e["end"]))] == ["A"]
|
||||
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], family="A")] == ["A"]
|
||||
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
|
||||
|
||||
class Answer:
|
||||
def __init__(self, r):
|
||||
self.r = r
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps(self.r).encode()
|
||||
|
||||
def fake_urlopen(*a, **k):
|
||||
r = next(answers)
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return Answer(r)
|
||||
before = urllib.request.urlopen, globals()["start_ollama"]
|
||||
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
|
||||
try:
|
||||
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
|
||||
finally:
|
||||
urllib.request.urlopen, globals()["start_ollama"] = before
|
||||
print("ok")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
|
||||
p.add_argument("--k", type=int, default=8, help="number of precedents (8)")
|
||||
p.add_argument("--family", help="only this family (key of families.md)")
|
||||
p.add_argument("--exclude", nargs=3, metavar=("STATION", "START", "END"),
|
||||
help="the range you analyse: its labels are left out")
|
||||
p.add_argument("--index", action="store_true", help="updates the index and stops")
|
||||
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
|
||||
a = p.parse_args()
|
||||
if a.check:
|
||||
return check()
|
||||
everything = list(labels())
|
||||
vectors = updated_index(everything)
|
||||
if a.index:
|
||||
return print(f"{len(vectors)} labels indexed ({MODEL}): {INDEX}")
|
||||
if not a.query:
|
||||
p.error("give a text, or --index")
|
||||
for score, e in search(a.query, everything, vectors, embed([a.query])[0], a.k, a.family, a.exclude):
|
||||
print(f"## {e.get('id', 'no id')} — {e.get('family') or 'no family'} — {e['station']} {e['start']} → {e['end']}\n"
|
||||
f"similarity {score:.2f}; {e['provenance']}\n{e['description']}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Validation of the automatic labels (labeling/auto/*.jsonl) against the rules of labeling/labeling-rules.md.
|
||||
|
||||
python3 labeling/scripts/validate.py # every station
|
||||
python3 labeling/scripts/validate.py NORDEND # one station
|
||||
python3 labeling/scripts/validate.py <folder>/labels.jsonl # a proposed file, before it is submitted (no `review` required)
|
||||
|
||||
Errors (to fix): unreadable JSON, missing field, unknown family, malformed date, start >= end,
|
||||
label without `review` or without `id` (`python3 labeling/scripts/ids.py`), id not first in the line, duplicate id, exact duplicate,
|
||||
numeric value, local time, "probably not a movement". "Nothing to worry about" is right only with an established cause: to
|
||||
judge by reading, not by script.
|
||||
Flagged (to look at): overlaps of two labels of the same family on the same station.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/labeling/auto, or through the labeling/ link
|
||||
|
||||
FAMILIES = {"Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"}
|
||||
DATE = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\dZ$")
|
||||
# A decimal comma (names such as F4.2 and dates such as 05.10 carry a dot), or a number followed by a unit.
|
||||
NUMBER = re.compile(r"(?<![\w.])\d+,\d+|\b\d+\s*(noise|noises|px|°|%|MAD|mm|times)(?!\w)|\b\d+\s+(targets|measurements) (out )?of\b")
|
||||
FORBIDDEN = re.compile(r"local time|local morning|local evening|local day|probably not a movement", re.I)
|
||||
|
||||
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
|
||||
errors, flagged, total = [], [], 0
|
||||
for st in stations:
|
||||
seen, ids, labels = set(), set(), []
|
||||
proposed = st.endswith(".jsonl") # a proposed file: its lines have no `review` nor `id` yet
|
||||
path = st if proposed else os.path.join(AUTO, f"{st}.jsonl")
|
||||
for n, line in enumerate(open(path, encoding="utf-8"), 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
total += 1
|
||||
where = f"{st}#{n}"
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
errors.append(f"{where} unreadable JSON"); continue
|
||||
missing = ({"start", "end", "family", "description"} | (set() if proposed else {"review", "id"})) - e.keys()
|
||||
if missing:
|
||||
errors.append(f"{where} missing field: {sorted(missing)}")
|
||||
if "id" in e:
|
||||
if next(iter(e)) != "id":
|
||||
errors.append(f"{where} id not first in the line (`python3 labeling/scripts/ids.py`)")
|
||||
if e["id"] in ids:
|
||||
errors.append(f"{where} duplicate id: {e['id']}")
|
||||
ids.add(e["id"])
|
||||
if e.get("family") not in FAMILIES:
|
||||
errors.append(f"{where} unknown family: {e.get('family')}")
|
||||
if not (DATE.match(e.get("start", "")) and DATE.match(e.get("end", ""))):
|
||||
errors.append(f"{where} malformed date"); continue
|
||||
if e["start"] >= e["end"]:
|
||||
errors.append(f"{where} start >= end ({e['start']})")
|
||||
key = (e["start"], e["end"], e["family"], e["description"])
|
||||
if key in seen:
|
||||
errors.append(f"{where} exact duplicate")
|
||||
seen.add(key)
|
||||
d = e.get("description", "")
|
||||
if m := NUMBER.search(d):
|
||||
errors.append(f"{where} numeric value: \"{m.group(0)}\"")
|
||||
if m := FORBIDDEN.search(d):
|
||||
errors.append(f"{where} forbidden: \"{m.group(0)}\"")
|
||||
labels.append((where, e))
|
||||
for i, (a, x) in enumerate(labels):
|
||||
for b, y in labels[i + 1:]:
|
||||
if x["family"] == y["family"] and x["start"] < y["end"] and y["start"] < x["end"]:
|
||||
flagged.append(f"{a} and {b}: two {x['family']} that overlap")
|
||||
|
||||
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
|
||||
for l in errors:
|
||||
print(" ERROR", l)
|
||||
for l in flagged[:40]:
|
||||
print(" to look at", l)
|
||||
if len(flagged) > 40:
|
||||
print(f" … and {len(flagged) - 40} other overlaps")
|
||||
sys.exit(1 if errors else 0)
|
||||
Reference in new issue
Block a user