"""Stable identifier of the automatic labels (D-131): `-`, given once, never changed nor reused. python3 labeling/scripts/ids.py # gives an id to the labels that have none, every station of labeling/auto/ python3 labeling/scripts/ids.py NORDEND # one station A label keeps its id when it is changed; an added label receives the next one of its station. The id is always the first field of the line. The rest does not change (order of the lines, other fields). Standard library. """ import glob import json import os import sys AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/aimsight-labeling/auto, or through the labeling/ link def id_first(e): """Puts the id back as the first field of the label, in place (lines are read by their id).""" if "id" in e and next(iter(e)) != "id": rest = {k: v for k, v in e.items() if k != "id"} ident = e["id"] e.clear() e.update({"id": ident}, **rest) def assign(station, labels, removed=()): """Gives an id to the labels that have none, after the ids of the station, and puts each id first -> the number of ids given. `removed`: the ids removed in the same operation, which are not given again.""" # ponytail: the next of the largest id present or removed in the operation; removing the last label, then adding one # later, in another operation, reuses its number. A counter per station if that becomes a problem. taken = [int(i.rsplit("-", 1)[1]) for i in [e["id"] for e in labels if "id" in e] + list(removed)] n = max(taken, default=0) given = 0 for e in labels: if "id" not in e: n += 1 e["id"] = f"{station}-{n}" given += 1 id_first(e) return given def update_file(station): p = os.path.join(AUTO, f"{station}.jsonl") lines = [l for l in open(p, encoding="utf-8") if l.strip()] labels = [json.loads(l) for l in lines] assign(station, labels) new = [json.dumps(e, ensure_ascii=False) + "\n" for e in labels] if new != [l if l.endswith("\n") else l + "\n" for l in lines]: # an id given or put back first tmp = p + ".tmp" with open(tmp, "w", encoding="utf-8") as f: f.writelines(new) os.replace(tmp, p) return labels if __name__ == "__main__": stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl"))) for st in stations: before = sum(1 for l in open(os.path.join(AUTO, f"{st}.jsonl"), encoding="utf-8") if l.strip() and '"id"' not in l) labels = update_file(st) ids = [e["id"] for e in labels] assert len(ids) == len(set(ids)), f"{st}: duplicate ids" print(f"{st}: {len(labels)} labels, {before} ids given")