2026-11-16 12:40:03 +01:00
|
|
|
"""Validation of the automatic labels (labeling/auto/*.jsonl) against the rules of labeling/labeling-rules.md.
|
|
|
|
|
|
2026-11-16 13:20:31 +01:00
|
|
|
python3 labeling/scripts/validate.py # every station
|
|
|
|
|
python3 labeling/scripts/validate.py NORDEND # one station
|
2026-11-17 10:01:22 +01:00
|
|
|
python3 labeling/scripts/validate.py <folder>/labels.jsonl # a proposed file, before it is submitted (no `id` required)
|
2026-11-18 10:51:31 +01:00
|
|
|
python3 labeling/scripts/validate.py cases # only the cases of auto/cases/ (checked also with no argument)
|
|
|
|
|
python3 labeling/scripts/validate.py --check # the self-test of the case check
|
|
|
|
|
|
|
|
|
|
A case (auto/cases/<id>.json, docs/superpowers/specs/2026-11-18-cases-design.md): known fields only, `id` = file name,
|
|
|
|
|
`kind` and `family` known, `name` and `signature` not empty, neighbours other existing cases with their `tell_apart`,
|
|
|
|
|
sources not empty and each a label id of auto/ or `user`, `expert`, `client report`, `review` a date; no station name and no
|
|
|
|
|
numeric value in its text.
|
2026-11-17 10:01:22 +01:00
|
|
|
|
|
|
|
|
`review` (optional): the date the user validated the label in aimsight-app; any change to the label removes it.
|
2026-11-16 12:40:03 +01:00
|
|
|
|
|
|
|
|
Errors (to fix): unreadable JSON, missing field, unknown family, malformed date, start >= end,
|
2026-11-17 10:01:22 +01:00
|
|
|
label without `id` (`python3 labeling/scripts/ids.py`), id not first in the line, duplicate id, exact duplicate,
|
2026-11-16 12:40:03 +01:00
|
|
|
numeric value, local time, "probably not a movement". "Nothing to worry about" is right only with an established cause: to
|
|
|
|
|
judge by reading, not by script.
|
|
|
|
|
Flagged (to look at): overlaps of two labels of the same family on the same station.
|
|
|
|
|
"""
|
|
|
|
|
import glob
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import re
|
|
|
|
|
import sys
|
|
|
|
|
|
2026-11-16 14:56:41 +01:00
|
|
|
AUTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "auto") # ~/aimsight-labeling/auto, or through the labeling/ link
|
2026-11-16 12:40:03 +01:00
|
|
|
|
2026-11-18 10:19:02 +01:00
|
|
|
FAMILIES = {"Visibility", "Obstruction", "Thermal displacement", "Displacement", "Perturbation", "Target issue", "No data"}
|
2026-11-16 12:40:03 +01:00
|
|
|
DATE = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\dZ$")
|
|
|
|
|
# A decimal comma (names such as F4.2 and dates such as 05.10 carry a dot), or a number followed by a unit.
|
|
|
|
|
NUMBER = re.compile(r"(?<![\w.])\d+,\d+|\b\d+\s*(noise|noises|px|°|%|MAD|mm|times)(?!\w)|\b\d+\s+(targets|measurements) (out )?of\b")
|
|
|
|
|
FORBIDDEN = re.compile(r"local time|local morning|local evening|local day|probably not a movement", re.I)
|
2026-11-18 10:51:31 +01:00
|
|
|
CASES = os.path.join(AUTO, "cases") # the cases: one JSON file per case (dotfiles are the build's own state)
|
|
|
|
|
KINDS = {"phenomenon", "false friend", "lesson"}
|
|
|
|
|
NON_LABEL = {"user", "expert", "client report"} # sources that are not a label id
|
|
|
|
|
CASE_FIELDS = {"id", "kind", "name", "family", "signature", "checks", "neighbours", "client", "sources", "review"}
|
|
|
|
|
CASE_REQUIRED = CASE_FIELDS - {"client", "review"}
|
|
|
|
|
DAY = re.compile(r"^\d{4}-\d\d-\d\d$")
|
|
|
|
|
LABEL_ID = re.compile(r"^[A-Z][A-Z0-9_]*-\d+$") # STATION-n
|
2026-11-16 12:40:03 +01:00
|
|
|
|
|
|
|
|
|
2026-11-17 09:32:20 +01:00
|
|
|
def check(e, proposed=False):
|
|
|
|
|
"""The errors of one label (a dict) against the rules, without those of its file (duplicates, overlaps) -> [text].
|
2026-11-17 10:01:22 +01:00
|
|
|
proposed: a label not yet submitted, without `id`. For a caller (aimsight-app: a label written in auto/)."""
|
2026-11-17 09:32:20 +01:00
|
|
|
errors = []
|
2026-11-17 10:01:22 +01:00
|
|
|
missing = ({"start", "end", "family", "description"} | (set() if proposed else {"id"})) - e.keys()
|
2026-11-17 09:32:20 +01:00
|
|
|
if missing:
|
|
|
|
|
errors.append(f"missing field: {sorted(missing)}")
|
|
|
|
|
if "id" in e and next(iter(e)) != "id":
|
|
|
|
|
errors.append("id not first in the line (`python3 labeling/scripts/ids.py`)")
|
|
|
|
|
if e.get("family") not in FAMILIES:
|
|
|
|
|
errors.append(f"unknown family: {e.get('family')}")
|
|
|
|
|
if not (DATE.match(e.get("start") or "") and DATE.match(e.get("end") or "")):
|
|
|
|
|
return errors + ["malformed date"]
|
|
|
|
|
if e["start"] >= e["end"]:
|
|
|
|
|
errors.append(f"start >= end ({e['start']})")
|
|
|
|
|
d = e.get("description") or ""
|
|
|
|
|
if "description" in e and not d.strip():
|
|
|
|
|
errors.append("empty description")
|
|
|
|
|
if m := NUMBER.search(d):
|
|
|
|
|
errors.append(f"numeric value: \"{m.group(0)}\"")
|
|
|
|
|
if m := FORBIDDEN.search(d):
|
|
|
|
|
errors.append(f"forbidden: \"{m.group(0)}\"")
|
|
|
|
|
return errors
|
|
|
|
|
|
|
|
|
|
|
2026-11-18 10:51:31 +01:00
|
|
|
def check_case(c, name, case_ids, label_ids, stations):
|
|
|
|
|
"""The errors of one case of auto/cases/ (a dict; `name` the stem of its file) -> [text]. A case is general: no station
|
|
|
|
|
name in its text, no numeric value; where it was seen is in its `sources`."""
|
|
|
|
|
errors = []
|
|
|
|
|
if missing := CASE_REQUIRED - c.keys():
|
|
|
|
|
errors.append(f"missing field: {sorted(missing)}")
|
|
|
|
|
if unknown := c.keys() - CASE_FIELDS:
|
|
|
|
|
errors.append(f"unknown field: {sorted(unknown)}")
|
|
|
|
|
if c.get("id") != name:
|
|
|
|
|
errors.append(f"id differs from the file name: {c.get('id')}")
|
|
|
|
|
if "kind" in c and c["kind"] not in KINDS:
|
|
|
|
|
errors.append(f"unknown kind: {c['kind']}")
|
|
|
|
|
if c.get("family") is not None and c["family"] not in FAMILIES:
|
|
|
|
|
errors.append(f"unknown family: {c['family']}")
|
|
|
|
|
for f in ("name", "signature"):
|
|
|
|
|
if f in c and not (isinstance(c[f], str) and c[f].strip()):
|
|
|
|
|
errors.append(f"empty {f}")
|
|
|
|
|
texts = [c.get("name"), c.get("signature"), c.get("client"), *(c.get("checks") or [])]
|
|
|
|
|
for n in c.get("neighbours") or []:
|
|
|
|
|
if n.get("case") not in case_ids or n.get("case") == c.get("id"):
|
|
|
|
|
errors.append(f"neighbour not another case: {n.get('case')}")
|
|
|
|
|
if not (n.get("tell_apart") or "").strip():
|
|
|
|
|
errors.append(f"neighbour without tell_apart: {n.get('case')}")
|
|
|
|
|
texts.append(n.get("tell_apart"))
|
|
|
|
|
if "sources" in c and not c["sources"]:
|
|
|
|
|
errors.append("no source")
|
|
|
|
|
for s in c.get("sources") or []: # a label id gone from auto/ is flagged by check_cases, not an error here
|
|
|
|
|
if s not in NON_LABEL and not LABEL_ID.match(str(s)):
|
|
|
|
|
errors.append(f"source neither a label id nor {sorted(NON_LABEL)}: {s}")
|
|
|
|
|
if "review" in c and not DAY.match(str(c["review"])):
|
|
|
|
|
errors.append(f"malformed review: {c['review']}")
|
|
|
|
|
for t in filter(None, texts):
|
|
|
|
|
if m := NUMBER.search(t):
|
|
|
|
|
errors.append(f"numeric value: \"{m.group(0)}\"")
|
|
|
|
|
for st in stations:
|
|
|
|
|
if re.search(rf"\b{re.escape(st)}\b", t):
|
|
|
|
|
errors.append(f"station name: {st}")
|
|
|
|
|
return errors
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_cases(auto=AUTO):
|
|
|
|
|
"""Every case of <auto>/cases/ -> (number of cases, [error], [to look at]). A label source no longer in auto/ (deleted
|
|
|
|
|
by aimsight-app since the last refresh) is to look at, not an error: the next `/cases` removes it."""
|
|
|
|
|
stations = sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(auto, "*.jsonl")))
|
|
|
|
|
label_ids = set()
|
|
|
|
|
for st in stations:
|
|
|
|
|
for line in open(os.path.join(auto, f"{st}.jsonl"), encoding="utf-8"):
|
|
|
|
|
try:
|
|
|
|
|
label_ids.add(json.loads(line).get("id"))
|
|
|
|
|
except json.JSONDecodeError:
|
|
|
|
|
pass # reported with the labels
|
|
|
|
|
found, errors, flagged = {}, [], []
|
|
|
|
|
for p in sorted(glob.glob(os.path.join(auto, "cases", "*.json"))): # glob skips the dotfiles
|
|
|
|
|
name = os.path.basename(p)[:-5]
|
|
|
|
|
try:
|
|
|
|
|
found[name] = json.load(open(p, encoding="utf-8"))
|
|
|
|
|
except json.JSONDecodeError:
|
|
|
|
|
errors.append(f"cases/{name}.json unreadable JSON")
|
|
|
|
|
for name, c in found.items():
|
|
|
|
|
errors += [f"cases/{name}.json {e}" for e in check_case(c, name, set(found), label_ids, set(stations))]
|
|
|
|
|
flagged += [f"cases/{name}.json source gone from auto/: {s}" for s in c.get("sources") or []
|
|
|
|
|
if s not in NON_LABEL and LABEL_ID.match(str(s)) and s not in label_ids]
|
|
|
|
|
return len(found) + sum(1 for e in errors if e.endswith("unreadable JSON")), errors, flagged
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def self_check():
|
|
|
|
|
ok = {"id": "a", "kind": "phenomenon", "name": "Targets jump together", "family": None,
|
|
|
|
|
"signature": "All the targets jump at one measurement.", "checks": ["overview.py --rigid"],
|
|
|
|
|
"neighbours": [{"case": "b", "tell_apart": "b lasts for hours"}], "sources": ["NORDEND-1", "user"]}
|
|
|
|
|
args = ({"a", "b"}, {"NORDEND-1"}, {"NORDEND"})
|
|
|
|
|
assert check_case(ok, "a", *args) == []
|
|
|
|
|
assert check_case({**ok, "review": "2026-11-20", "client": "Nothing to do."}, "a", *args) == []
|
|
|
|
|
|
|
|
|
|
def bad(change, name="a"):
|
|
|
|
|
return check_case({**ok, **change}, name, *args)
|
|
|
|
|
assert bad({}, name="x") # id differs from the file name
|
|
|
|
|
assert bad({"kind": "rule"}) and bad({"family": "Rain"})
|
|
|
|
|
assert bad({"signature": ""}) and bad({"name": ""})
|
|
|
|
|
assert bad({"neighbours": [{"case": "zzz", "tell_apart": "x"}]}) # unknown neighbour
|
|
|
|
|
assert bad({"neighbours": [{"case": "a", "tell_apart": "x"}]}) # itself
|
|
|
|
|
assert bad({"neighbours": [{"case": "b", "tell_apart": ""}]})
|
|
|
|
|
assert bad({"sources": []}) and bad({"sources": ["n/a"]}) # empty, neither a label id nor a known source
|
|
|
|
|
assert check_case({**ok, "sources": ["NORDEND-9"]}, "a", *args) == [] # a label gone from auto/: flagged, not an error
|
|
|
|
|
import tempfile
|
|
|
|
|
with tempfile.TemporaryDirectory() as d:
|
|
|
|
|
os.makedirs(os.path.join(d, "cases"))
|
|
|
|
|
with open(os.path.join(d, "NORDEND.jsonl"), "w") as f:
|
|
|
|
|
f.write(json.dumps({"id": "NORDEND-1"}) + "\n{broken\n") # an unreadable label line: no traceback
|
|
|
|
|
for c in ({**ok, "neighbours": [], "sources": ["NORDEND-1", "NORDEND-9"]},):
|
|
|
|
|
with open(os.path.join(d, "cases", "a.json"), "w") as f:
|
|
|
|
|
json.dump(c, f)
|
|
|
|
|
with open(os.path.join(d, "cases", ".read.json"), "w") as f:
|
|
|
|
|
f.write("{}")
|
|
|
|
|
n, errs, flagged = check_cases(d)
|
|
|
|
|
assert (n, errs) == (1, []) and flagged == ["cases/a.json source gone from auto/: NORDEND-9"], (n, errs, flagged)
|
|
|
|
|
assert bad({"signature": "At NORDEND the targets jump."}) # station name
|
|
|
|
|
assert bad({"signature": "A jump of 3 mm."}) # numeric value
|
|
|
|
|
assert bad({"stations": ["NORDEND"]}) # unknown field
|
|
|
|
|
assert bad({"review": "yesterday"})
|
|
|
|
|
label = {"start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "Perturbation", "description": "x"}
|
|
|
|
|
assert not any("unknown family" in e for e in check(label, proposed=True))
|
|
|
|
|
print("ok")
|
|
|
|
|
|
|
|
|
|
|
2026-11-17 09:32:20 +01:00
|
|
|
def main():
|
2026-11-18 10:51:31 +01:00
|
|
|
if sys.argv[1:] == ["--check"]:
|
|
|
|
|
return self_check()
|
|
|
|
|
with_cases = sys.argv[1:] in ([], ["cases"])
|
|
|
|
|
stations = [] if sys.argv[1:] == ["cases"] else (
|
|
|
|
|
sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl"))))
|
2026-11-17 09:32:20 +01:00
|
|
|
errors, flagged, total = [], [], 0
|
|
|
|
|
for st in stations:
|
|
|
|
|
seen, ids, labels = set(), set(), []
|
2026-11-17 10:01:22 +01:00
|
|
|
proposed = st.endswith(".jsonl") # a proposed file: its lines have no `id` yet
|
2026-11-17 09:32:20 +01:00
|
|
|
path = st if proposed else os.path.join(AUTO, f"{st}.jsonl")
|
|
|
|
|
for n, line in enumerate(open(path, encoding="utf-8"), 1):
|
|
|
|
|
if not line.strip():
|
|
|
|
|
continue
|
|
|
|
|
total += 1
|
|
|
|
|
where = f"{st}#{n}"
|
|
|
|
|
try:
|
|
|
|
|
e = json.loads(line)
|
|
|
|
|
except json.JSONDecodeError:
|
|
|
|
|
errors.append(f"{where} unreadable JSON"); continue
|
|
|
|
|
found = check(e, proposed)
|
|
|
|
|
if "id" in e:
|
|
|
|
|
if e["id"] in ids:
|
|
|
|
|
found.append(f"duplicate id: {e['id']}")
|
|
|
|
|
ids.add(e["id"])
|
|
|
|
|
errors += [f"{where} {f}" for f in found]
|
|
|
|
|
if "malformed date" in found:
|
|
|
|
|
continue
|
|
|
|
|
key = (e["start"], e["end"], e.get("family"), e.get("description"))
|
|
|
|
|
if key in seen:
|
|
|
|
|
errors.append(f"{where} exact duplicate")
|
|
|
|
|
seen.add(key)
|
|
|
|
|
labels.append((where, e))
|
|
|
|
|
for i, (a, x) in enumerate(labels):
|
|
|
|
|
for b, y in labels[i + 1:]:
|
|
|
|
|
if x.get("family") == y.get("family") and x["start"] < y["end"] and y["start"] < x["end"]:
|
|
|
|
|
flagged.append(f"{a} and {b}: two {x.get('family')} that overlap")
|
|
|
|
|
|
2026-11-18 10:51:31 +01:00
|
|
|
if stations:
|
|
|
|
|
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
|
|
|
|
|
for l in errors:
|
|
|
|
|
print(" ERROR", l)
|
|
|
|
|
for l in flagged[:40]:
|
|
|
|
|
print(" to look at", l)
|
|
|
|
|
if len(flagged) > 40:
|
|
|
|
|
print(f" … and {len(flagged) - 40} other overlaps")
|
|
|
|
|
if with_cases:
|
|
|
|
|
n, case_errors, gone = check_cases()
|
|
|
|
|
print(f"{n} cases: {len(case_errors)} errors, {len(gone)} label sources gone from auto/ to look at (`/cases` removes them)")
|
|
|
|
|
for l in case_errors:
|
|
|
|
|
print(" ERROR", l)
|
|
|
|
|
for l in gone:
|
|
|
|
|
print(" to look at", l)
|
|
|
|
|
errors += case_errors
|
2026-11-17 09:32:20 +01:00
|
|
|
sys.exit(1 if errors else 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|