English: label map, families, labeling rules, knowledge/, validate.py (ex controle.py), ids.py, precedents.py; the labels of auto/ translated, keys start/end/family/review/analysis (manual/: keys only); analysis folders renamed; the 2026-11-16 trial analysis (NORDEND-243)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 12:40:03 +01:00
1 parent 4d926ecddc
commit 68a0eefd8b
79 files changed
+2789 -2652

No files matched your search

+104 -104
View File
@@ -1,17 +1,17 @@
"""Les précédents : les étiquettes de auto/ les plus proches d'un texte, par le sens (RAG). manual/ est à l'utilisateur
seul : aucun processus ne s'en sert.
"""The precedents: the labels of auto/ closest to a text, by meaning (RAG). manual/ is the user's alone: no process
uses it.
L'analyste décrit ce qu'il voit sur sa plage ; l'outil rend les étiquettes déjà posées qui en parlent le plus, toutes
stations, avec leur provenance (relue, non relue) : à lui de juger si elles valent pour son cas.
The analyst describes what it sees on its range; the tool returns the labels already written that speak of it most, all
stations, with their provenance (reviewed, not reviewed): it is up to the analyst to judge whether they fit its case.
python3 labeling/precedents.py "les cibles sautent ensemble quand une référence est rejetée par intermittence"
python3 labeling/precedents.py "<texte>" --exclure NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --famille Obstruction --k 5
python3 labeling/precedents.py --index # met l'index à jour seulement
python3 labeling/precedents.py --check # contrôle de la logique, sans Ollama
python3 labeling/precedents.py "the targets jump together when a reference is rejected intermittently"
python3 labeling/precedents.py "<text>" --exclude NORDEND 2026-11-04T21:12Z 2026-11-07T14:12Z --family Obstruction --k 5
python3 labeling/precedents.py --index # only updates the index
python3 labeling/precedents.py --check # checks the logic, without Ollama
L'index (.index/, hors de git) garde le vecteur de chaque texte : à chaque appel, seules les étiquettes nouvelles ou
modifiées passent par le modèle. Modèle : bge-m3 servi par Ollama, le conteneur ollama-bge-m3, que l'outil démarre
lui-même s'il ne répond pas (`docker start`) ; OLLAMA_URL et RAG_MODELE pour en changer. Bibliothèque standard seulement.
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed labels go through
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
"""
import argparse
import glob
@@ -26,59 +26,59 @@ import urllib.request
from datetime import datetime
from pathlib import Path
ICI = Path(__file__).resolve().parent
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # le port publié par le conteneur ollama-bge-m3
MODELE = os.environ.get("RAG_MODELE", "bge-m3:567m")
CONTENEUR = "ollama-bge-m3" # démarré par l'outil s'il ne répond pas
INDEX = ICI / ".index" / f"{MODELE.replace(':', '_')}.json"
LOT = 32 # textes par appel au modèle
HERE = Path(__file__).resolve().parent
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
INDEX = HERE / ".index" / f"{MODEL.replace(':', '_')}.json"
BATCH = 32 # texts per call to the model
def etiquettes(racine=ICI):
"""Toutes les étiquettes de auto/, avec leur station et leur provenance ; jamais celles de manual/."""
for f in sorted(glob.glob(str(racine / "auto" / "*.jsonl"))):
def labels(root=HERE):
"""Every label of auto/, with its station and its provenance; never those of manual/."""
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
station = Path(f).stem
for ligne in open(f, encoding="utf-8"):
if ligne.strip():
e = json.loads(ligne)
for line in open(f, encoding="utf-8"):
if line.strip():
e = json.loads(line)
e["station"], e["provenance"] = station, provenance(e)
yield e
def provenance(e):
if e.get("revue"):
return f"auto, relue le {e['revue']}"
return "auto, non relue" + (f" (analyse {e['analyse']})" if e.get("analyse") else "")
if e.get("review"):
return f"auto, reviewed on {e['review']}"
return "auto, not reviewed" + (f" (analysis {e['analysis']})" if e.get("analysis") else "")
def texte(e):
return f"{e.get('categorie', '')}. {e.get('description', '')}"
def text(e):
return f"{e.get('family', '')}. {e.get('description', '')}"
def cle(t):
def key(t):
return hashlib.sha1(t.encode("utf-8")).hexdigest()
def instant(s):
def parse_time(s):
return datetime.fromisoformat(s.replace("Z", "+00:00"))
def recouvre(e, station, debut, fin):
"""L'étiquette touche-t-elle la plage exclue (même station, intervalles qui se chevauchent) ?"""
return e["station"] == station and instant(e["debut"]) <= instant(fin) and instant(e["fin"]) >= instant(debut)
def overlaps(e, station, start, end):
"""Does the label touch the excluded range (same station, overlapping intervals)?"""
return e["station"] == station and parse_time(e["start"]) <= parse_time(end) and parse_time(e["end"]) >= parse_time(start)
def norme(v):
def norm(v):
n = math.sqrt(sum(x * x for x in v)) or 1.0
return [round(x / n, 5) for x in v]
def demarrer_ollama(attente=120):
"""Le conteneur ollama-bge-m3 arrêté : `docker start`, puis attendre qu'il réponde -> True s'il répond."""
if subprocess.run(["docker", "start", CONTENEUR], capture_output=True).returncode:
def start_ollama(wait=120):
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
return False
fin = time.monotonic() + attente
while time.monotonic() < fin:
deadline = time.monotonic() + wait
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
return True
@@ -87,62 +87,62 @@ def demarrer_ollama(attente=120):
return False
def vectoriser(textes):
corps = json.dumps({"model": MODELE, "input": textes}).encode()
req = urllib.request.Request(f"{URL}/api/embed", corps, {"Content-Type": "application/json"})
for essai in (1, 2): # Ollama arrêté : on démarre son conteneur, une fois, puis on relance
def embed(texts):
body = json.dumps({"model": MODEL, "input": texts}).encode()
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
try:
with urllib.request.urlopen(req, timeout=300) as r:
return [norme(v) for v in json.load(r)["embeddings"]]
return [norm(v) for v in json.load(r)["embeddings"]]
except OSError as err:
if essai == 2 or not demarrer_ollama():
sys.exit(f"Ollama injoignable sur {URL} ({err}) : `docker start {CONTENEUR}`, ou OLLAMA_URL.")
if attempt == 2 or not start_ollama():
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
def index_a_jour(toutes):
"""Les vecteurs de toutes les étiquettes ; calcule ceux qui manquent, oublie ceux des textes disparus."""
vecteurs = json.loads(INDEX.read_text()) if INDEX.exists() else {}
voulus = {cle(texte(e)): texte(e) for e in toutes}
manquants = [k for k in voulus if k not in vecteurs]
for i in range(0, len(manquants), LOT):
lot = manquants[i:i + LOT]
vecteurs.update(zip(lot, vectoriser([voulus[k] for k in lot])))
print(f"index : {min(i + LOT, len(manquants))}/{len(manquants)} nouveaux textes", file=sys.stderr)
if manquants or len(vecteurs) != len(voulus):
vecteurs = {k: vecteurs[k] for k in voulus}
def updated_index(everything):
"""The vectors of every label; computes the missing ones, forgets those of vanished texts."""
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
wanted = {key(text(e)): text(e) for e in everything}
missing = [k for k in wanted if k not in vectors]
for i in range(0, len(missing), BATCH):
batch = missing[i:i + BATCH]
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
if missing or len(vectors) != len(wanted):
vectors = {k: vectors[k] for k in wanted}
INDEX.parent.mkdir(exist_ok=True)
INDEX.write_text(json.dumps(vecteurs))
return vecteurs
INDEX.write_text(json.dumps(vectors))
return vectors
def chercher(requete, toutes, vecteurs, q, k=8, famille=None, exclure=None):
retenues = [e for e in toutes if (not famille or e.get("categorie") == famille) and not (exclure and recouvre(e, *exclure))]
notes = [(sum(a * b for a, b in zip(q, vecteurs[cle(texte(e))])), e) for e in retenues]
return sorted(notes, key=lambda n: -n[0])[:k]
def search(query, everything, vectors, q, k=8, family=None, exclude=None):
kept = [e for e in everything if (not family or e.get("family") == family) and not (exclude and overlaps(e, *exclude))]
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(e))])), e) for e in kept]
return sorted(scores, key=lambda n: -n[0])[:k]
def check():
e = {"station": "S", "debut": "2026-11-04T00:00Z", "fin": "2026-11-05T00:00Z"}
assert recouvre(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
assert recouvre(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # bord commun : exclue
assert not recouvre(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z")
assert not recouvre(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
assert provenance({"revue": "2026-11-04"}) == "auto, relue le 2026-11-04"
import tempfile # seules les étiquettes de auto/ servent : manual/ est à l'utilisateur seul
e = {"station": "S", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"}
assert overlaps(e, "S", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
assert overlaps(e, "S", "2026-11-03T00:00Z", "2026-11-04T00:00Z") # common edge: excluded
assert not overlaps(e, "S", "2026-11-05T00:01Z", "2026-11-06T00:00Z")
assert not overlaps(e, "T", "2026-11-04T12:00Z", "2026-11-06T00:00Z")
assert provenance({"review": "2026-11-04"}) == "auto, reviewed on 2026-11-04"
import tempfile # only the labels of auto/ are used: manual/ is the user's alone
with tempfile.TemporaryDirectory() as d:
for sous, desc in (("manual", "de l'utilisateur"), ("auto", "d'un agent")):
(Path(d) / sous).mkdir()
(Path(d) / sous / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8")
assert [x["description"] for x in etiquettes(Path(d))] == ["d'un agent"]
assert provenance({"analyse": "X"}) == "auto, non relue (analyse X)"
a, b = {**e, "categorie": "A", "description": "a"}, {**e, "station": "T", "categorie": "B", "description": "b"}
vec = {cle(texte(a)): [1.0, 0.0], cle(texte(b)): [0.6, 0.8]}
assert [x["categorie"] for _, x in chercher("", [a, b], vec, [0.0, 1.0])] == ["B", "A"]
assert [x["categorie"] for _, x in chercher("", [a, b], vec, [0.0, 1.0], exclure=("T", e["debut"], e["fin"]))] == ["A"]
assert [x["categorie"] for _, x in chercher("", [a, b], vec, [0.0, 1.0], famille="A")] == ["A"]
appels, reponses = [], iter([OSError("refusé"), {"embeddings": [[3.0, 4.0]]}]) # Ollama arrêté : démarré, puis la relance passe
for sub, desc in (("manual", "the user's"), ("auto", "an agent's")):
(Path(d) / sub).mkdir()
(Path(d) / sub / "S.jsonl").write_text(json.dumps({**e, "description": desc}) + "\n", encoding="utf-8")
assert [x["description"] for x in labels(Path(d))] == ["an agent's"]
assert provenance({"analysis": "X"}) == "auto, not reviewed (analysis X)"
a, b = {**e, "family": "A", "description": "a"}, {**e, "station": "T", "family": "B", "description": "b"}
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0])] == ["B", "A"]
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], exclude=("T", e["start"], e["end"]))] == ["A"]
assert [x["family"] for _, x in search("", [a, b], vec, [0.0, 1.0], family="A")] == ["A"]
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
class Reponse:
class Answer:
def __init__(self, r):
self.r = r
@@ -155,41 +155,41 @@ def check():
def read(self):
return json.dumps(self.r).encode()
def ouvrir(*a, **k):
r = next(reponses)
def fake_urlopen(*a, **k):
r = next(answers)
if isinstance(r, Exception):
raise r
return Reponse(r)
avant = urllib.request.urlopen, globals()["demarrer_ollama"]
urllib.request.urlopen, globals()["demarrer_ollama"] = ouvrir, lambda: appels.append(1) or True
return Answer(r)
before = urllib.request.urlopen, globals()["start_ollama"]
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
try:
assert vectoriser(["x"]) == [[0.6, 0.8]] and appels == [1], appels
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
finally:
urllib.request.urlopen, globals()["demarrer_ollama"] = avant
urllib.request.urlopen, globals()["start_ollama"] = before
print("ok")
def main():
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("requete", nargs="?", help="ce que tu vois, en français, comme une description d'étiquette")
p.add_argument("--k", type=int, default=8, help="nombre de précédents (8)")
p.add_argument("--famille", help="seulement cette famille (clé de familles.md)")
p.add_argument("--exclure", nargs=3, metavar=("STATION", "DEBUT", "FIN"),
help="la plage que tu analyses : ses étiquettes sont écartées")
p.add_argument("--index", action="store_true", help="met l'index à jour et s'arrête")
p.add_argument("--check", action="store_true", help="contrôle de la logique, sans Ollama")
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
p.add_argument("--k", type=int, default=8, help="number of precedents (8)")
p.add_argument("--family", help="only this family (key of families.md)")
p.add_argument("--exclude", nargs=3, metavar=("STATION", "START", "END"),
help="the range you analyse: its labels are left out")
p.add_argument("--index", action="store_true", help="updates the index and stops")
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
a = p.parse_args()
if a.check:
return check()
toutes = list(etiquettes())
vecteurs = index_a_jour(toutes)
everything = list(labels())
vectors = updated_index(everything)
if a.index:
return print(f"{len(vecteurs)} étiquettes indexées ({MODELE}) : {INDEX}")
if not a.requete:
p.error("donne un texte, ou --index")
for note, e in chercher(a.requete, toutes, vecteurs, vectoriser([a.requete])[0], a.k, a.famille, a.exclure):
print(f"## {e.get('id', 'sans id')} — {e.get('categorie') or 'sans famille'} — {e['station']} {e['debut']} → {e['fin']}\n"
f"similarité {note:.2f} ; {e['provenance']}\n{e['description']}\n")
return print(f"{len(vectors)} labels indexed ({MODEL}): {INDEX}")
if not a.query:
p.error("give a text, or --index")
for score, e in search(a.query, everything, vectors, embed([a.query])[0], a.k, a.family, a.exclude):
print(f"## {e.get('id', 'no id')} — {e.get('family') or 'no family'} — {e['station']} {e['start']} → {e['end']}\n"
f"similarity {score:.2f}; {e['provenance']}\n{e['description']}\n")
if __name__ == "__main__":