Files

64 lines
2.8 KiB
Python
Raw Permalink Normal View History

"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
python3 labeling/scripts/ids.py # gives an id to the labels that have none, every station of labeling/auto/
python3 labeling/scripts/ids.py NORDEND # one station
A label keeps its id when it is changed; an added label receives the next one of its station.
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
"""
import glob
import json
import os
import sys
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/aimsight-labeling/auto, or through the labeling/ link
def id_first(e):
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
2026-11-05 08:58:47 +01:00
if "id" in e and next(iter(e)) != "id":
rest = {k: v for k, v in e.items() if k != "id"}
2026-11-05 08:58:47 +01:00
ident = e["id"]
e.clear()
e.update({"id": ident}, **rest)
2026-11-05 08:58:47 +01:00
def assign(station, labels, removed=()):
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
n = max(taken, default=0)
given = 0
for e in labels:
if "id" not in e:
n += 1
e["id"] = f"{station}-{n}"
given += 1
id_first(e)
return given
def update_file(station):
p = os.path.join(AUTO, f"{station}.jsonl")
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
labels = [json.loads(l) for l in lines]
assign(station, labels)
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
tmp = p + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
f.writelines(new)
os.replace(tmp, p)
return labels
if __name__ == "__main__":
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
for st in stations:
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
labels = update_file(st)
ids = [e["id"] for e in labels]
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
print(f"{st}: {len(labels)} labels, {before} ids given")