The labeling scripts in scripts/ (validate, ids, precedents); the index stays in .index/
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4688d8ab9e
commit
f34616ae38
6 files changed
+22
-22
No files matched your search
@@ -0,0 +1,64 @@
|
||||
"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
|
||||
|
||||
python3 labeling/scripts/ids.py # gives an id to the labels that have none, every station of labeling/auto/
|
||||
python3 labeling/scripts/ids.py NORDEND # one station
|
||||
|
||||
A label keeps its id when it is changed; an added label receives the next one of its station.
|
||||
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/labeling/auto, or through the labeling/ link
|
||||
|
||||
|
||||
def id_first(e):
|
||||
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
|
||||
if "id" in e and next(iter(e)) != "id":
|
||||
rest = {k: v for k, v in e.items() if k != "id"}
|
||||
ident = e["id"]
|
||||
e.clear()
|
||||
e.update({"id": ident}, **rest)
|
||||
|
||||
|
||||
def assign(station, labels, removed=()):
|
||||
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
|
||||
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
|
||||
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
|
||||
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
|
||||
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
|
||||
n = max(taken, default=0)
|
||||
given = 0
|
||||
for e in labels:
|
||||
if "id" not in e:
|
||||
n += 1
|
||||
e["id"] = f"{station}-{n}"
|
||||
given += 1
|
||||
id_first(e)
|
||||
return given
|
||||
|
||||
|
||||
def update_file(station):
|
||||
p = os.path.join(AUTO, f"{station}.jsonl")
|
||||
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
|
||||
labels = [json.loads(l) for l in lines]
|
||||
assign(station, labels)
|
||||
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
|
||||
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
|
||||
tmp = p + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.writelines(new)
|
||||
os.replace(tmp, p)
|
||||
return labels
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
|
||||
for st in stations:
|
||||
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
|
||||
labels = update_file(st)
|
||||
ids = [e["id"] for e in labels]
|
||||
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
|
||||
print(f"{st}: {len(labels)} labels, {before} ids given")
|
||||
Reference in new issue
Block a user