For aimsight-app: validate.check (the rules of one label), client.py (what the client is told of an event, from families.md); families.md: For the client per variety

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 09:32:20 +01:00
1 parent 83f92bb7b9
commit 542e91ccb2
5 files changed
+130 -54

No files matched your search

+71 -52
View File
@@ -24,56 +24,75 @@ DATE = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\dZ$")
NUMBER = re.compile(r"(?<![\w.])\d+,\d+|\b\d+\s*(noise|noises|px|°|%|MAD|mm|times)(?!\w)|\b\d+\s+(targets|measurements) (out )?of\b")
FORBIDDEN = re.compile(r"local time|local morning|local evening|local day|probably not a movement", re.I)
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
errors, flagged, total = [], [], 0
for st in stations:
seen, ids, labels = set(), set(), []
proposed = st.endswith(".jsonl") # a proposed file: its lines have no `review` nor `id` yet
path = st if proposed else os.path.join(AUTO, f"{st}.jsonl")
for n, line in enumerate(open(path, encoding="utf-8"), 1):
if not line.strip():
continue
total += 1
where = f"{st}#{n}"
try:
e = json.loads(line)
except json.JSONDecodeError:
errors.append(f"{where} unreadable JSON"); continue
missing = ({"start", "end", "family", "description"} | (set() if proposed else {"review", "id"})) - e.keys()
if missing:
errors.append(f"{where} missing field: {sorted(missing)}")
if "id" in e:
if next(iter(e)) != "id":
errors.append(f"{where} id not first in the line (`python3 labeling/scripts/ids.py`)")
if e["id"] in ids:
errors.append(f"{where} duplicate id: {e['id']}")
ids.add(e["id"])
if e.get("family") not in FAMILIES:
errors.append(f"{where} unknown family: {e.get('family')}")
if not (DATE.match(e.get("start", "")) and DATE.match(e.get("end", ""))):
errors.append(f"{where} malformed date"); continue
if e["start"] >= e["end"]:
errors.append(f"{where} start >= end ({e['start']})")
key = (e["start"], e["end"], e["family"], e["description"])
if key in seen:
errors.append(f"{where} exact duplicate")
seen.add(key)
d = e.get("description", "")
if m := NUMBER.search(d):
errors.append(f"{where} numeric value: \"{m.group(0)}\"")
if m := FORBIDDEN.search(d):
errors.append(f"{where} forbidden: \"{m.group(0)}\"")
labels.append((where, e))
for i, (a, x) in enumerate(labels):
for b, y in labels[i + 1:]:
if x["family"] == y["family"] and x["start"] < y["end"] and y["start"] < x["end"]:
flagged.append(f"{a} and {b}: two {x['family']} that overlap")
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
for l in errors:
print(" ERROR", l)
for l in flagged[:40]:
print(" to look at", l)
if len(flagged) > 40:
print(f" … and {len(flagged) - 40} other overlaps")
sys.exit(1 if errors else 0)
def check(e, proposed=False):
"""The errors of one label (a dict) against the rules, without those of its file (duplicates, overlaps) -> [text].
proposed: a label not yet submitted, without `review` nor `id`. For a caller (aimsight-app: a label written in auto/)."""
errors = []
missing = ({"start", "end", "family", "description"} | (set() if proposed else {"review", "id"})) - e.keys()
if missing:
errors.append(f"missing field: {sorted(missing)}")
if "id" in e and next(iter(e)) != "id":
errors.append("id not first in the line (`python3 labeling/scripts/ids.py`)")
if e.get("family") not in FAMILIES:
errors.append(f"unknown family: {e.get('family')}")
if not (DATE.match(e.get("start") or "") and DATE.match(e.get("end") or "")):
return errors + ["malformed date"]
if e["start"] >= e["end"]:
errors.append(f"start >= end ({e['start']})")
d = e.get("description") or ""
if "description" in e and not d.strip():
errors.append("empty description")
if m := NUMBER.search(d):
errors.append(f"numeric value: \"{m.group(0)}\"")
if m := FORBIDDEN.search(d):
errors.append(f"forbidden: \"{m.group(0)}\"")
return errors
def main():
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
errors, flagged, total = [], [], 0
for st in stations:
seen, ids, labels = set(), set(), []
proposed = st.endswith(".jsonl") # a proposed file: its lines have no `review` nor `id` yet
path = st if proposed else os.path.join(AUTO, f"{st}.jsonl")
for n, line in enumerate(open(path, encoding="utf-8"), 1):
if not line.strip():
continue
total += 1
where = f"{st}#{n}"
try:
e = json.loads(line)
except json.JSONDecodeError:
errors.append(f"{where} unreadable JSON"); continue
found = check(e, proposed)
if "id" in e:
if e["id"] in ids:
found.append(f"duplicate id: {e['id']}")
ids.add(e["id"])
errors += [f"{where} {f}" for f in found]
if "malformed date" in found:
continue
key = (e["start"], e["end"], e.get("family"), e.get("description"))
if key in seen:
errors.append(f"{where} exact duplicate")
seen.add(key)
labels.append((where, e))
for i, (a, x) in enumerate(labels):
for b, y in labels[i + 1:]:
if x.get("family") == y.get("family") and x["start"] < y["end"] and y["start"] < x["end"]:
flagged.append(f"{a} and {b}: two {x.get('family')} that overlap")
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
for l in errors:
print(" ERROR", l)
for l in flagged[:40]:
print(" to look at", l)
if len(flagged) > 40:
print(f" … and {len(flagged) - 40} other overlaps")
sys.exit(1 if errors else 0)
if __name__ == "__main__":
main()