65 lines
2.8 KiB
Python
65 lines
2.8 KiB
Python
"""Stable identifier of the automatic labels (D-131): `<STATION>-<n>`, given once, never changed nor reused.
|
|
|
|
python3 labeling/scripts/ids.py # gives an id to the labels that have none, every station of labeling/auto/
|
|
python3 labeling/scripts/ids.py NORDEND # one station
|
|
|
|
A label keeps its id when it is changed; an added label receives the next one of its station.
|
|
The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library.
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/aimsight-labeling/auto, or through the labeling/ link
|
|
|
|
|
|
def id_first(e):
|
|
"""Puts the id back as the first field of the label, in place (lines are read by their id)."""
|
|
if "id" in e and next(iter(e)) != "id":
|
|
rest = {k: v for k, v in e.items() if k != "id"}
|
|
ident = e["id"]
|
|
e.clear()
|
|
e.update({"id": ident}, **rest)
|
|
|
|
|
|
def assign(station, labels, removed=()):
|
|
"""Gives an id to the labels that have none, after the ids of the station, and puts each id first
|
|
-> the number of ids given. `removed`: the ids removed in the same operation, which are not given again."""
|
|
# ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one
|
|
# later, in another operation, reuses its number. A counter per station if that becomes a problem.
|
|
taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)]
|
|
n = max(taken, default=0)
|
|
given = 0
|
|
for e in labels:
|
|
if "id" not in e:
|
|
n += 1
|
|
e["id"] = f"{station}-{n}"
|
|
given += 1
|
|
id_first(e)
|
|
return given
|
|
|
|
|
|
def update_file(station):
|
|
p = os.path.join(AUTO, f"{station}.jsonl")
|
|
lines = [l for l in open(p, encoding="utf-8") if l.strip()]
|
|
labels = [json.loads(l) for l in lines]
|
|
assign(station, labels)
|
|
new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels]
|
|
if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first
|
|
tmp = p + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
f.writelines(new)
|
|
os.replace(tmp, p)
|
|
return labels
|
|
|
|
|
|
if __name__ == "__main__":
|
|
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
|
|
for st in stations:
|
|
before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l)
|
|
labels = update_file(st)
|
|
ids = [e["id"] for e in labels]
|
|
assert len(ids) == len(set(ids)), f"{st}: duplicate ids"
|
|
print(f"{st}: {len(labels)} labels, {before} ids given")
|