79 lines
3.9 KiB
Python
79 lines
3.9 KiB
Python
"""Validation of the automatic labels (labeling/auto/*.jsonl) against the rules of labeling/labeling-rules.md.
|
|||
|
|
|
||
|
|
python3 labeling/validate.py # every station
|
||
|
|
python3 labeling/validate.py NORDEND # one station
|
||
|
|
python3 labeling/validate.py <folder>/labels.jsonl # a proposed file, before it is submitted (no `review` required)
|
||
|
|
|
||
|
|
Errors (to fix): unreadable JSON, missing field, unknown family, malformed date, start >= end,
|
||
|
|
label without `review` or without `id` (`python3 labeling/ids.py`), id not first in the line, duplicate id, exact duplicate,
|
||
|
|
numeric value, local time, "probably not a movement". "Nothing to worry about" is right only with an established cause: to
|
||
|
|
judge by reading, not by script.
|
||
|
|
Flagged (to look at): overlaps of two labels of the same family on the same station.
|
||
|
|
"""
|
||
|
|
import glob
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
|
||
|
|
AUTO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "auto") # ~/labeling/auto, or through the labeling/ link
|
||
|
|
|
||
|
|
FAMILIES = {"Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"}
|
||
|
|
DATE = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\dZ$")
|
||
|
|
# A decimal comma (names such as F4.2 and dates such as 05.10 carry a dot), or a number followed by a unit.
|
||
|
|
NUMBER = re.compile(r"(?<![\w.])\d+,\d+|\b\d+\s*(noise|noises|px|°|%|MAD|mm|times)(?!\w)|\b\d+\s+(targets|measurements) (out )?of\b")
|
||
|
|
FORBIDDEN = re.compile(r"local time|local morning|local evening|local day|probably not a movement", re.I)
|
||
|
|
|
||
|
|
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
|
||
|
|
errors, flagged, total = [], [], 0
|
||
|
|
for st in stations:
|
||
|
|
seen, ids, labels = set(), set(), []
|
||
|
|
proposed = st.endswith(".jsonl") # a proposed file: its lines have no `review` nor `id` yet
|
||
|
|
path = st if proposed else os.path.join(AUTO, f"{st}.jsonl")
|
||
|
|
for n, line in enumerate(open(path, encoding="utf-8"), 1):
|
||
|
|
if not line.strip():
|
||
|
|
continue
|
||
|
|
total += 1
|
||
|
|
where = f"{st}#{n}"
|
||
|
|
try:
|
||
|
|
e = json.loads(line)
|
||
|
|
except json.JSONDecodeError:
|
||
|
|
errors.append(f"{where} unreadable JSON"); continue
|
||
|
|
missing = ({"start", "end", "family", "description"} | (set() if proposed else {"review", "id"})) - e.keys()
|
||
|
|
if missing:
|
||
|
|
errors.append(f"{where} missing field: {sorted(missing)}")
|
||
|
|
if "id" in e:
|
||
|
|
if next(iter(e)) != "id":
|
||
|
|
errors.append(f"{where} id not first in the line (`python3 labeling/ids.py`)")
|
||
|
|
if e["id"] in ids:
|
||
|
|
errors.append(f"{where} duplicate id: {e['id']}")
|
||
|
|
ids.add(e["id"])
|
||
|
|
if e.get("family") not in FAMILIES:
|
||
|
|
errors.append(f"{where} unknown family: {e.get('family')}")
|
||
|
|
if not (DATE.match(e.get("start", "")) and DATE.match(e.get("end", ""))):
|
||
|
|
errors.append(f"{where} malformed date"); continue
|
||
|
|
if e["start"] >= e["end"]:
|
||
|
|
errors.append(f"{where} start >= end ({e['start']})")
|
||
|
|
key = (e["start"], e["end"], e["family"], e["description"])
|
||
|
|
if key in seen:
|
||
|
|
errors.append(f"{where} exact duplicate")
|
||
|
|
seen.add(key)
|
||
|
|
d = e.get("description", "")
|
||
|
|
if m := NUMBER.search(d):
|
||
|
|
errors.append(f"{where} numeric value: \"{m.group(0)}\"")
|
||
|
|
if m := FORBIDDEN.search(d):
|
||
|
|
errors.append(f"{where} forbidden: \"{m.group(0)}\"")
|
||
|
|
labels.append((where, e))
|
||
|
|
for i, (a, x) in enumerate(labels):
|
||
|
|
for b, y in labels[i + 1:]:
|
||
|
|
if x["family"] == y["family"] and x["start"] < y["end"] and y["start"] < x["end"]:
|
||
|
|
flagged.append(f"{a} and {b}: two {x['family']} that overlap")
|
||
|
|
|
||
|
|
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
|
||
|
|
for l in errors:
|
||
|
|
print(" ERROR", l)
|
||
|
|
for l in flagged[:40]:
|
||
|
|
print(" to look at", l)
|
||
|
|
if len(flagged) > 40:
|
||
|
|
print(f" … and {len(flagged) - 40} other overlaps")
|
||
|
|
sys.exit(1 if errors else 0)
|