English: label map, families, labeling rules, knowledge/, validate.py (ex controle.py), ids.py, precedents.py; the labels of auto/ translated, keys start/end/family/review/analysis (manual/: keys only); analysis folders renamed; the 2026-11-16 trial analysis (NORDEND-243)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4d926ecddc
commit
68a0eefd8b
79 files changed
+2789
-2652
No files matched your search
@@ -1,64 +1,64 @@
|
||||
"""Identifiant stable des étiquettes automatiques (D-131) : `<STATION>-<n>`, donné une fois, jamais changé ni réutilisé.
|
||||
"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
|
||||
|
||||
python3 labeling/ids.py # donne un id aux étiquettes qui n'en ont pas, toutes les stations de labeling/auto/
|
||||
python3 labeling/ids.py NORDEND # une station
|
||||
python3 labeling/ids.py # gives an id to the labels that have none, every station of labeling/auto/
|
||||
python3 labeling/ids.py NORDEND # one station
|
||||
|
||||
Une étiquette garde son id quand on la modifie ; une étiquette ajoutée reçoit le suivant de sa station.
|
||||
L'id est toujours le premier champ de la ligne. Le reste ne change pas (ordre des lignes, autres champs). Bibliothèque standard.
|
||||
A label keeps its id when it is changed; an added label receives the next one of its station.
|
||||
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
AUTO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "auto") # ~/labeling/auto, ou par le lien labeling/
|
||||
AUTO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "auto") # ~/labeling/auto, or through the labeling/ link
|
||||
|
||||
|
||||
def en_tete(e):
|
||||
"""Remet l'id en premier champ de l'étiquette, en place (les lignes se lisent par leur id)."""
|
||||
def id_first(e):
|
||||
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
|
||||
if "id" in e and next(iter(e)) != "id":
|
||||
reste = {k: v for k, v in e.items() if k != "id"}
|
||||
rest = {k: v for k, v in e.items() if k != "id"}
|
||||
ident = e["id"]
|
||||
e.clear()
|
||||
e.update({"id": ident}, **reste)
|
||||
e.update({"id": ident}, **rest)
|
||||
|
||||
|
||||
def attribuer(station, etiquettes, retires=()):
|
||||
"""Donne un id aux étiquettes qui n'en ont pas, à la suite des ids de la station, et met chaque id en tête
|
||||
-> le nombre d'ids donnés. `retires` : les ids retirés dans la même opération, qui ne se redonnent pas."""
|
||||
# ponytail: le suivant du plus grand id présent ou retiré dans l'opération ; retirer la dernière étiquette, puis en
|
||||
# ajouter une plus tard, dans une autre opération, réutilise son numéro. Un compteur par station si cela gêne.
|
||||
pris = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in etiquettes if "id" in e] + list(retires)]
|
||||
n = max(pris, default=0)
|
||||
donnes = 0
|
||||
for e in etiquettes:
|
||||
def assign(station, labels, removed=()):
|
||||
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
|
||||
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
|
||||
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
|
||||
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
|
||||
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
|
||||
n = max(taken, default=0)
|
||||
given = 0
|
||||
for e in labels:
|
||||
if "id" not in e:
|
||||
n += 1
|
||||
e["id"] = f"{station}-{n}"
|
||||
donnes += 1
|
||||
en_tete(e)
|
||||
return donnes
|
||||
given += 1
|
||||
id_first(e)
|
||||
return given
|
||||
|
||||
|
||||
def fichier(station):
|
||||
def update_file(station):
|
||||
p = os.path.join(AUTO, f"{station}.jsonl")
|
||||
lignes = [l for l in open(p, encoding="utf-8") if l.strip()]
|
||||
es = [json.loads(l) for l in lignes]
|
||||
attribuer(station, es)
|
||||
neuves = [json.dumps(e, ensure_ascii=False) + "\n" for e in es]
|
||||
if neuves != [l if l.endswith("\n") else l + "\n" for l in lignes]: # un id donné ou remis en tête
|
||||
provisoire = p + ".tmp"
|
||||
with open(provisoire, "w", encoding="utf-8") as f:
|
||||
f.writelines(neuves)
|
||||
os.replace(provisoire, p)
|
||||
return es
|
||||
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
|
||||
labels = [json.loads(l) for l in lines]
|
||||
assign(station, labels)
|
||||
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
|
||||
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
|
||||
tmp = p + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.writelines(new)
|
||||
os.replace(tmp, p)
|
||||
return labels
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
|
||||
for st in stations:
|
||||
avant = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
|
||||
es = fichier(st)
|
||||
ids = [e["id"] for e in es]
|
||||
assert len(ids) == len(set(ids)), f"{st} : ids en double"
|
||||
print(f"{st} : {len(es)} étiquettes, {avant} ids donnés")
|
||||
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
|
||||
labels = update_file(st)
|
||||
ids = [e["id"] for e in labels]
|
||||
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
|
||||
print(f"{st}: {len(labels)} labels, {before} ids given")
|
||||
Reference in new issue
Block a user