English: label map, families, labeling rules, knowledge/, validate.py (ex controle.py), ids.py, precedents.py; the labels of auto/ translated, keys start/end/family/review/analysis (manual/: keys only); analysis folders renamed; the 2026-11-16 trial analysis (NORDEND-243)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 12:40:03 +01:00
1 parent 4d926ecddc
commit 68a0eefd8b
79 files changed
+2789 -2652

No files matched your search

+38 -38
View File
@@ -1,64 +1,64 @@
"""Identifiant stable des étiquettes automatiques (D-131) : `<STATION>-<n>`, donné une fois, jamais changé ni réutilisé.
"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
python3 labeling/ids.py # donne un id aux étiquettes qui n'en ont pas, toutes les stations de labeling/auto/
python3 labeling/ids.py NORDEND # une station
python3 labeling/ids.py # gives an id to the labels that have none, every station of labeling/auto/
python3 labeling/ids.py NORDEND # one station
Une étiquette garde son id quand on la modifie ; une étiquette ajoutée reçoit le suivant de sa station.
L'id est toujours le premier champ de la ligne. Le reste ne change pas (ordre des lignes, autres champs). Bibliothèque standard.
A label keeps its id when it is changed; an added label receives the next one of its station.
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
"""
import glob
import json
import os
import sys
AUTO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "auto") # ~/labeling/auto, ou par le lien labeling/
AUTO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "auto") # ~/labeling/auto, or through the labeling/ link
def en_tete(e):
"""Remet l'id en premier champ de l'étiquette, en place (les lignes se lisent par leur id)."""
def id_first(e):
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
if "id" in e and next(iter(e)) != "id":
reste = {k: v for k, v in e.items() if k != "id"}
rest = {k: v for k, v in e.items() if k != "id"}
ident = e["id"]
e.clear()
e.update({"id": ident}, **reste)
e.update({"id": ident}, **rest)
def attribuer(station, etiquettes, retires=()):
"""Donne un id aux étiquettes qui n'en ont pas, à la suite des ids de la station, et met chaque id en tête
-> le nombre d'ids donnés. `retires` : les ids retirés dans la même opération, qui ne se redonnent pas."""
# ponytail: le suivant du plus grand id présent ou retiré dans l'opération ; retirer la dernière étiquette, puis en
# ajouter une plus tard, dans une autre opération, réutilise son numéro. Un compteur par station si cela gêne.
pris = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in etiquettes if "id" in e] + list(retires)]
n = max(pris, default=0)
donnes = 0
for e in etiquettes:
def assign(station, labels, removed=()):
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
n = max(taken, default=0)
given = 0
for e in labels:
if "id" not in e:
n += 1
e["id"] = f"{station}-{n}"
donnes += 1
en_tete(e)
return donnes
given += 1
id_first(e)
return given
def fichier(station):
def update_file(station):
p = os.path.join(AUTO, f"{station}.jsonl")
lignes = [l for l in open(p, encoding="utf-8") if l.strip()]
es = [json.loads(l) for l in lignes]
attribuer(station, es)
neuves = [json.dumps(e, ensure_ascii=False) + "\n" for e in es]
if neuves != [l if l.endswith("\n") else l + "\n" for l in lignes]: # un id donné ou remis en tête
provisoire = p + ".tmp"
with open(provisoire, "w", encoding="utf-8") as f:
f.writelines(neuves)
os.replace(provisoire, p)
return es
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
labels = [json.loads(l) for l in lines]
assign(station, labels)
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
tmp = p + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
f.writelines(new)
os.replace(tmp, p)
return labels
if __name__ == "__main__":
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
for st in stations:
avant = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
es = fichier(st)
ids = [e["id"] for e in es]
assert len(ids) == len(set(ids)), f"{st} : ids en double"
print(f"{st} : {len(es)} étiquettes, {avant} ids donnés")
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
labels = update_file(st)
ids = [e["id"] for e in labels]
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
print(f"{st}: {len(labels)} labels, {before} ids given")