This commit is contained in:
maxime committed 2026-11-18 10:51:31 +01:00
1 parent 8e4ebca342
commit 58b630f2d5
112 files changed
+3172 -295

No files matched your search

+396
View File
@@ -0,0 +1,396 @@
"""The cases: the phenomena, false friends and lessons drawn from the labels, one JSON file each in auto/cases/ (the
design: docs/superpowers/specs/2026-11-18-cases-design.md); the RAG that finds those closest to a text, and the
bookkeeping of their refresh by the `cases` skill (.claude/skills/cases/). manual/ is the user's alone: no process uses it.
The analyst describes what it sees on its range; the tool returns the cases that speak of it most, each with its
signature, its checks, its neighbours and how to tell them apart, its sources and whether the user validated it: it is
up to the analyst to judge whether they fit.
python3 labeling/scripts/cases.py "the targets jump together when a reference is rejected intermittently"
python3 labeling/scripts/cases.py "<text>" --family Obstruction --k 5
python3 labeling/scripts/cases.py --list # every case, one line each, without Ollama
python3 labeling/scripts/cases.py --index # only updates the index
python3 labeling/scripts/cases.py --check # checks the logic, without Ollama
python3 labeling/scripts/cases.py --pending DIR # the refresh: the labels to read, one file per family, in DIR
python3 labeling/scripts/cases.py --record FILE # the refresh: {"assignments": {label id: [case ids]}} applied
auto/cases/.read.json, the build's state: per label id, the hash of the text it was read with and the cases it went to.
The index (.index/, out of git) keeps the vector of each text: at each call, only the new or changed cases go through
the model. Model: bge-m3 served by Ollama, the ollama-bge-m3 container, which the tool starts by itself if it does not
answer (`docker start`); OLLAMA_URL and RAG_MODEL to change them. Standard library only.
"""
import argparse
import glob
import hashlib
import json
import math
import os
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
HERE = Path(__file__).resolve().parent.parent # the labeling root: auto/ and .index/
URL = os.environ.get("OLLAMA_URL", "http://10.0.2.140:11435") # the port published by the ollama-bge-m3 container
MODEL = os.environ.get("RAG_MODEL", "bge-m3:567m")
CONTAINER = "ollama-bge-m3" # started by the tool if it does not answer
INDEX = HERE / ".index" / f"cases_{MODEL.replace(':', '_')}.json"
BATCH = 32 # texts per call to the model
def cases(root=HERE):
"""Every case of auto/cases/ (not the dotfiles, the build's state), sorted by id."""
out = []
for f in sorted(glob.glob(str(root / "auto" / "cases" / "*.json"))):
try:
out.append(json.loads(Path(f).read_text(encoding="utf-8")))
except json.JSONDecodeError:
sys.exit(f"{f}: unreadable JSON")
return sorted(out, key=lambda c: c["id"])
def labels(root=HERE):
"""Every label of auto/ by id, with its station; never those of manual/."""
out = {}
for f in sorted(glob.glob(str(root / "auto" / "*.jsonl"))):
for line in open(f, encoding="utf-8"):
if line.strip():
e = json.loads(line)
out[e["id"]] = e | {"station": Path(f).stem}
return out
def provenance(c):
return f"validated by the user on {c['review']}" if c.get("review") else "not validated"
def text(c):
return f"{c['kind']}. {c.get('family') or ''}. {c['name']}. {c['signature']}"
def listing(cs):
return [f"{c['id']} | {c['kind']} | {c.get('family') or '-'} | {c['name']}" for c in cs]
def key(t):
return hashlib.sha1(t.encode("utf-8")).hexdigest()
def norm(v):
n = math.sqrt(sum(x * x for x in v)) or 1.0
return [round(x / n, 5) for x in v]
def start_ollama(wait=120):
"""The ollama-bge-m3 container stopped: `docker start`, then wait until it answers -> True if it answers."""
if subprocess.run(["docker", "start", CONTAINER], capture_output=True).returncode:
return False
deadline = time.monotonic() + wait
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"{URL}/api/version", timeout=5):
return True
except OSError:
time.sleep(2)
return False
def embed(texts):
body = json.dumps({"model": MODEL, "input": texts}).encode()
req = urllib.request.Request(f"{URL}/api/embed", body, {"Content-Type": "application/json"})
for attempt in (1, 2): # Ollama stopped: its container is started, once, then the call is retried
try:
with urllib.request.urlopen(req, timeout=300) as r:
return [norm(v) for v in json.load(r)["embeddings"]]
except OSError as err:
if attempt == 2 or not start_ollama():
sys.exit(f"Ollama unreachable at {URL} ({err}): `docker start {CONTAINER}`, or OLLAMA_URL.")
def updated_index(everything):
"""The vectors of every case; computes the missing ones, forgets those of vanished texts."""
vectors = json.loads(INDEX.read_text()) if INDEX.exists() else {}
wanted = {key(text(e)): text(e) for e in everything}
missing = [k for k in wanted if k not in vectors]
for i in range(0, len(missing), BATCH):
batch = missing[i:i + BATCH]
vectors.update(zip(batch, embed([wanted[k] for k in batch])))
print(f"index: {min(i + BATCH, len(missing))}/{len(missing)} new texts", file=sys.stderr)
if missing or len(vectors) != len(wanted):
vectors = {k: vectors[k] for k in wanted}
INDEX.parent.mkdir(exist_ok=True)
INDEX.write_text(json.dumps(vectors))
return vectors
def search(q, everything, vectors, k=8, family=None):
"""The k cases closest to the query vector q (of `family` and the cross-family ones, if given) -> [(similarity, case)]."""
kept = [c for c in everything if not family or c.get("family") in (family, None)]
scores = [(sum(a * b for a, b in zip(q, vectors[key(text(c))])), c) for c in kept]
return sorted(scores, key=lambda n: -n[0])[:k]
def main_list(root=HERE):
for line in listing(cases(root)):
print(line)
def show(score, c, known):
"""One case found, in full, its label sources dated from `known` (the labels of auto/ by id)."""
def source(s):
e = known.get(s)
return f"{s} ({e['station']} {e['start']} → {e['end']})" if e else (s if s in ("user", "expert", "client report") else f"{s} (not in auto/)")
out = [f"## {c['id']} — {c['kind']} — {c.get('family') or 'no family'} — similarity {score:.2f}; {provenance(c)}",
c["name"], c["signature"]]
out += [f"check: {x}" for x in c.get("checks") or []]
out += [f"neighbour {n['case']}: {n['tell_apart']}" for n in c.get("neighbours") or []]
if c.get("client"):
out.append(f"client: {c['client']}")
out.append("sources: " + "; ".join(source(s) for s in c["sources"]))
return "\n".join(out) + "\n"
def label_hash(label):
"""What a label says, for the refresh: a label read again when its bounds, family or description change."""
return key(f"{label['start']} {label['end']} {label['family']}. {label['description']}")
def _read_path(root):
return root / "auto" / "cases" / ".read.json"
def _write(path, obj):
"""Writes a JSON file, readable in a diff, next to it then replaces it: an interruption does not damage it."""
tmp = path.with_name(path.name + ".tmp")
tmp.write_text(json.dumps(obj, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
os.replace(tmp, path)
def pending(root=HERE):
"""The labels to read, by family (new, changed, or attached to no existing case), and the ids deleted since the
last record -> ({family: [label]}, [id])."""
known, existing = labels(root), {c["id"] for c in cases(root)}
f = _read_path(root)
read = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {}
todo = {}
for i, e in known.items():
r = read.get(i)
if not r or r["hash"] != label_hash(e) or not r["cases"] or any(c not in existing for c in r["cases"]):
todo.setdefault(e["family"], []).append(e)
return todo, sorted(i for i in read if i not in known)
def write_pending(todo, out):
"""One file per family in `out`, `<family in lowercase, dashes>.jsonl`, one label per line. Removes first the files
of an earlier refresh it owns (labels, answers, assignments), and only those: a wrong `out` loses nothing else."""
from validate import FAMILIES
out.mkdir(parents=True, exist_ok=True)
for family in FAMILIES:
for suffix in (".jsonl", ".answer.json"):
(out / f"{family.lower().replace(' ', '-')}{suffix}").unlink(missing_ok=True)
(out / "assignments.json").unlink(missing_ok=True)
for family, ls in todo.items():
(out / f"{family.lower().replace(' ', '-')}.jsonl").write_text(
"".join(json.dumps(e, ensure_ascii=False) + "\n" for e in ls), encoding="utf-8")
def record(assignments, root=HERE):
"""Applies {label id: [case ids]} (the cases each label read illustrates) to the sources of the cases and to
.read.json; removes from every case the label sources no longer in auto/; deletes the cases left without a source
(a non-label source keeps a case). Never touches `review` -> {added, removed, deleted_cases, skipped}."""
from validate import NON_LABEL
known = labels(root)
before = {c["id"]: c for c in cases(root)}
cs = json.loads(json.dumps(before))
if unknown := sorted({i for v in assignments.values() for i in v if i not in cs}):
sys.exit(f"unknown case(s): {unknown}; nothing written")
f = _read_path(root)
read = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {}
added, removed = {}, {}
skipped = sorted(i for i in assignments if i not in known)
for c in cs.values():
for s in [s for s in c["sources"] if s not in NON_LABEL and s not in known]:
c["sources"].remove(s)
removed.setdefault(c["id"], []).append(s)
read = {i: r for i, r in read.items() if i in known}
for i, new in assignments.items():
if i in skipped:
continue
for c in cs.values():
if i in c["sources"] and c["id"] not in new:
c["sources"].remove(i)
removed.setdefault(c["id"], []).append(i)
for cid in new:
if i not in cs[cid]["sources"]:
cs[cid]["sources"].append(i)
added.setdefault(cid, []).append(i)
read[i] = {"hash": label_hash(known[i]), "cases": list(new)}
deleted = sorted(cid for cid, c in cs.items() if not c["sources"])
folder = root / "auto" / "cases"
for cid, c in cs.items():
if cid in deleted:
(folder / f"{cid}.json").unlink()
elif c != before[cid]:
_write(folder / f"{cid}.json", c)
_write(f, read)
return {"added": added, "removed": removed, "deleted_cases": deleted, "skipped": skipped}
def check():
import tempfile
with tempfile.TemporaryDirectory() as d:
root = Path(d)
(root / "auto" / "cases").mkdir(parents=True)
(root / "manual").mkdir()
a = {"id": "a", "kind": "phenomenon", "name": "A", "family": "Visibility", "signature": "a", "sources": ["S-1"]}
b = {"id": "b", "kind": "false friend", "name": "B", "family": None, "signature": "b", "sources": ["user"], "review": "2026-11-20"}
for c in (b, a):
(root / "auto" / "cases" / f"{c['id']}.json").write_text(json.dumps(c))
(root / "auto" / "cases" / ".read.json").write_text("{}")
(root / "manual" / "S.jsonl").write_text(json.dumps({"id": "S-1", "description": "the user's"}) + "\n")
(root / "auto" / "S.jsonl").write_text(json.dumps({"id": "S-1", "start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "Visibility", "description": "an agent's"}) + "\n")
assert [c["id"] for c in cases(root)] == ["a", "b"] # sorted, dotfile ignored
assert labels(root)["S-1"]["description"] == "an agent's" and labels(root)["S-1"]["station"] == "S"
assert listing(cases(root)) == ["a | phenomenon | Visibility | A", "b | false friend | - | B"]
assert provenance(b) == "validated by the user on 2026-11-20" and provenance(a) == "not validated"
vec = {key(text(a)): [1.0, 0.0], key(text(b)): [0.6, 0.8]}
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec)] == ["b", "a"]
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec, family="Visibility")] == ["b", "a"] # null kept
assert [c["id"] for _, c in search([0.0, 1.0], [a, b], vec, family="Obstruction")] == ["b"]
shown = show(0.5, {**a, "sources": ["S-1", "Z-9", "expert"]}, labels(root)).splitlines()
assert shown[0] == "## a — phenomenon — Visibility — similarity 0.50; not validated"
assert shown[1] == "A" # the name: a false friend's statement
assert shown[-1] == "sources: S-1 (S 2026-11-04T00:00Z → 2026-11-04T01:00Z); Z-9 (not in auto/); expert"
before = globals()["embed"] # --list without Ollama
globals()["embed"] = lambda *x: (_ for _ in ()).throw(AssertionError("--list called Ollama"))
try:
import contextlib, io
with contextlib.redirect_stdout(io.StringIO()) as out:
main_list(root)
assert out.getvalue() == "a | phenomenon | Visibility | A\nb | false friend | - | B\n"
finally:
globals()["embed"] = before
(root / "auto" / "cases" / "c.json").write_text("{broken")
try:
cases(root)
raise AssertionError("a broken case must stop with its path")
except SystemExit as e:
assert "c.json" in str(e)
with tempfile.TemporaryDirectory() as d: # the bookkeeping of a refresh
root = Path(d); cdir = root / "auto" / "cases"; cdir.mkdir(parents=True)
def put_labels(*ls):
(root / "auto" / "S.jsonl").write_text("".join(json.dumps(l) + "\n" for l in ls))
def case(i, sources, **kw):
(cdir / f"{i}.json").write_text(json.dumps({"id": i, "kind": "phenomenon", "name": i, "family": None, "signature": i, "sources": sources, **kw}))
def get(i):
return json.loads((cdir / f"{i}.json").read_text())
quiet = __import__("contextlib").redirect_stderr(__import__("io").StringIO())
lab = {"id": "S-1", "start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "No data", "description": "gap"}
(root / "auto" / "T.jsonl").write_text(json.dumps({**lab, "id": "T-5", "family": "Visibility", "description": "fog"}) + "\n")
put_labels(lab); case("c1", ["user"], review="2026-11-20"); case("c2", ["T-5"])
todo, deleted = pending(root)
assert sorted(l["id"] for ls in todo.values() for l in ls) == ["S-1", "T-5"] and deleted == []
r = record({"S-1": ["c1"], "T-5": ["c2"]}, root)
assert r["added"] == {"c1": ["S-1"]} and get("c1")["sources"] == ["user", "S-1"] # c2 had T-5 already
assert get("c1")["review"] == "2026-11-20" # a new source keeps the validation
assert pending(root) == ({}, [])
put_labels({**lab, "description": "longer gap"}) # same id, text changed: read again
assert [l["id"] for l in pending(root)[0]["No data"]] == ["S-1"]
record({"S-1": ["c2"]}, root)
assert get("c1")["sources"] == ["user"] and "S-1" in get("c2")["sources"]
record({"S-1": []}, root) # no case: stays to read
assert "S-1" in [l["id"] for l in pending(root)[0]["No data"]]
try:
with quiet:
record({"S-1": ["nope"]}, root)
raise AssertionError("unknown case must refuse")
except SystemExit:
pass
record({"S-1": ["c2"]}, root)
(cdir / "c2.json").unlink() # deleted by hand by the user
assert "S-1" in [l["id"] for l in pending(root)[0]["No data"]]
case("c3", ["S-1", "T-5"]); record({"S-1": ["c3"]}, root)
put_labels() # S-1 deleted by aimsight-app
assert pending(root)[1] == ["S-1"]
r = record({"S-1": ["c3"]}, root) # assigned after its deletion
assert r["skipped"] == ["S-1"] and get("c3")["sources"] == ["T-5"]
case("c4", ["S-2"]); r = record({}, root) # a seeded source never recorded, gone from auto/
assert r["deleted_cases"] == ["c4"] and not (cdir / "c4.json").exists()
assert (cdir / "c1.json").exists() and get("c3")["sources"] == ["T-5"] # non-label source: never deleted
pending_dir = root / "work"; write_pending({"No data": [lab]}, pending_dir)
assert (pending_dir / "no-data.jsonl").exists()
for name in ("S.jsonl", "visibility.answer.json", "assignments.json"):
(pending_dir / name).write_text("{}")
write_pending({}, pending_dir) # a wrong DIR must not lose station files
assert (pending_dir / "S.jsonl").exists()
assert not any((pending_dir / n).exists() for n in ("no-data.jsonl", "visibility.answer.json", "assignments.json"))
calls, answers = [], iter([OSError("refused"), {"embeddings": [[3.0, 4.0]]}]) # Ollama stopped: started, then the retry passes
class Answer:
def __init__(self, r):
self.r = r
def __enter__(self):
return self
def __exit__(self, *a):
return False
def read(self):
return json.dumps(self.r).encode()
def fake_urlopen(*a, **k):
r = next(answers)
if isinstance(r, Exception):
raise r
return Answer(r)
before = urllib.request.urlopen, globals()["start_ollama"]
urllib.request.urlopen, globals()["start_ollama"] = fake_urlopen, lambda: calls.append(1) or True
try:
assert embed(["x"]) == [[0.6, 0.8]] and calls == [1], calls
finally:
urllib.request.urlopen, globals()["start_ollama"] = before
print("ok")
def main():
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("query", nargs="?", help="what you see, in English, as a label description")
p.add_argument("--k", type=int, default=8, help="number of cases (8)")
p.add_argument("--family", help="only this family (key of families.md)")
p.add_argument("--list", action="store_true", help="every case, one line each, without Ollama")
p.add_argument("--index", action="store_true", help="updates the index and stops")
p.add_argument("--pending", metavar="DIR", help="writes the labels to read, one file per family, in DIR")
p.add_argument("--record", metavar="FILE", help='applies {"assignments": {label id: [case ids]}} from FILE')
p.add_argument("--check", action="store_true", help="checks the logic, without Ollama")
a = p.parse_args()
if a.check:
return check()
if a.list:
return main_list()
if a.pending:
todo, deleted = pending()
write_pending(todo, Path(a.pending))
for family, ls in sorted(todo.items()):
print(f"{family}: {len(ls)} labels to read")
return print(f"deleted since the last record: {deleted or 'none'}")
if a.record:
return print(json.dumps(record(json.loads(Path(a.record).read_text(encoding="utf-8"))["assignments"]), indent=1))
everything = cases()
vectors = updated_index(everything)
if a.index:
return print(f"{len(vectors)} cases indexed ({MODEL}): {INDEX}")
if not a.query:
p.error("give a text, --list or --index")
known = labels()
for score, c in search(embed([a.query])[0], everything, vectors, a.k, a.family):
print(show(score, c, known))
if __name__ == "__main__":
main()
+139 -8
View File
@@ -3,6 +3,13 @@
python3 labeling/scripts/validate.py # every station
python3 labeling/scripts/validate.py NORDEND # one station
python3 labeling/scripts/validate.py <folder>/labels.jsonl # a proposed file, before it is submitted (no `id` required)
python3 labeling/scripts/validate.py cases # only the cases of auto/cases/ (checked also with no argument)
python3 labeling/scripts/validate.py --check # the self-test of the case check
A case (auto/cases/<id>.json, docs/superpowers/specs/2026-11-18-cases-design.md): known fields only, `id` = file name,
`kind` and `family` known, `name` and `signature` not empty, neighbours other existing cases with their `tell_apart`,
sources not empty and each a label id of auto/ or `user`, `expert`, `client report`, `review` a date; no station name and no
numeric value in its text.
`review` (optional): the date the user validated the label in aimsight-app; any change to the label removes it.
@@ -25,6 +32,13 @@ DATE = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\dZ$")
# A decimal comma (names such as F4.2 and dates such as 05.10 carry a dot), or a number followed by a unit.
NUMBER = re.compile(r"(?<![\w.])\d+,\d+|\b\d+\s*(noise|noises|px|°|%|MAD|mm|times)(?!\w)|\b\d+\s+(targets|measurements) (out )?of\b")
FORBIDDEN = re.compile(r"local time|local morning|local evening|local day|probably not a movement", re.I)
CASES = os.path.join(AUTO, "cases") # the cases: one JSON file per case (dotfiles are the build's own state)
KINDS = {"phenomenon", "false friend", "lesson"}
NON_LABEL = {"user", "expert", "client report"} # sources that are not a label id
CASE_FIELDS = {"id", "kind", "name", "family", "signature", "checks", "neighbours", "client", "sources", "review"}
CASE_REQUIRED = CASE_FIELDS - {"client", "review"}
DAY = re.compile(r"^\d{4}-\d\d-\d\d$")
LABEL_ID = re.compile(r"^[A-Z][A-Z0-9_]*-\d+$") # STATION-n
def check(e, proposed=False):
@@ -52,8 +66,116 @@ def check(e, proposed=False):
return errors
def check_case(c, name, case_ids, label_ids, stations):
"""The errors of one case of auto/cases/ (a dict; `name` the stem of its file) -> [text]. A case is general: no station
name in its text, no numeric value; where it was seen is in its `sources`."""
errors = []
if missing := CASE_REQUIRED - c.keys():
errors.append(f"missing field: {sorted(missing)}")
if unknown := c.keys() - CASE_FIELDS:
errors.append(f"unknown field: {sorted(unknown)}")
if c.get("id") != name:
errors.append(f"id differs from the file name: {c.get('id')}")
if "kind" in c and c["kind"] not in KINDS:
errors.append(f"unknown kind: {c['kind']}")
if c.get("family") is not None and c["family"] not in FAMILIES:
errors.append(f"unknown family: {c['family']}")
for f in ("name", "signature"):
if f in c and not (isinstance(c[f], str) and c[f].strip()):
errors.append(f"empty {f}")
texts = [c.get("name"), c.get("signature"), c.get("client"), *(c.get("checks") or [])]
for n in c.get("neighbours") or []:
if n.get("case") not in case_ids or n.get("case") == c.get("id"):
errors.append(f"neighbour not another case: {n.get('case')}")
if not (n.get("tell_apart") or "").strip():
errors.append(f"neighbour without tell_apart: {n.get('case')}")
texts.append(n.get("tell_apart"))
if "sources" in c and not c["sources"]:
errors.append("no source")
for s in c.get("sources") or []: # a label id gone from auto/ is flagged by check_cases, not an error here
if s not in NON_LABEL and not LABEL_ID.match(str(s)):
errors.append(f"source neither a label id nor {sorted(NON_LABEL)}: {s}")
if "review" in c and not DAY.match(str(c["review"])):
errors.append(f"malformed review: {c['review']}")
for t in filter(None, texts):
if m := NUMBER.search(t):
errors.append(f"numeric value: \"{m.group(0)}\"")
for st in stations:
if re.search(rf"\b{re.escape(st)}\b", t):
errors.append(f"station name: {st}")
return errors
def check_cases(auto=AUTO):
"""Every case of <auto>/cases/ -> (number of cases, [error], [to look at]). A label source no longer in auto/ (deleted
by aimsight-app since the last refresh) is to look at, not an error: the next `/cases` removes it."""
stations = sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(auto, "*.jsonl")))
label_ids = set()
for st in stations:
for line in open(os.path.join(auto, f"{st}.jsonl"), encoding="utf-8"):
try:
label_ids.add(json.loads(line).get("id"))
except json.JSONDecodeError:
pass # reported with the labels
found, errors, flagged = {}, [], []
for p in sorted(glob.glob(os.path.join(auto, "cases", "*.json"))): # glob skips the dotfiles
name = os.path.basename(p)[:-5]
try:
found[name] = json.load(open(p, encoding="utf-8"))
except json.JSONDecodeError:
errors.append(f"cases/{name}.json unreadable JSON")
for name, c in found.items():
errors += [f"cases/{name}.json {e}" for e in check_case(c, name, set(found), label_ids, set(stations))]
flagged += [f"cases/{name}.json source gone from auto/: {s}" for s in c.get("sources") or []
if s not in NON_LABEL and LABEL_ID.match(str(s)) and s not in label_ids]
return len(found) + sum(1 for e in errors if e.endswith("unreadable JSON")), errors, flagged
def self_check():
ok = {"id": "a", "kind": "phenomenon", "name": "Targets jump together", "family": None,
"signature": "All the targets jump at one measurement.", "checks": ["overview.py --rigid"],
"neighbours": [{"case": "b", "tell_apart": "b lasts for hours"}], "sources": ["NORDEND-1", "user"]}
args = ({"a", "b"}, {"NORDEND-1"}, {"NORDEND"})
assert check_case(ok, "a", *args) == []
assert check_case({**ok, "review": "2026-11-20", "client": "Nothing to do."}, "a", *args) == []
def bad(change, name="a"):
return check_case({**ok, **change}, name, *args)
assert bad({}, name="x") # id differs from the file name
assert bad({"kind": "rule"}) and bad({"family": "Rain"})
assert bad({"signature": ""}) and bad({"name": ""})
assert bad({"neighbours": [{"case": "zzz", "tell_apart": "x"}]}) # unknown neighbour
assert bad({"neighbours": [{"case": "a", "tell_apart": "x"}]}) # itself
assert bad({"neighbours": [{"case": "b", "tell_apart": ""}]})
assert bad({"sources": []}) and bad({"sources": ["n/a"]}) # empty, neither a label id nor a known source
assert check_case({**ok, "sources": ["NORDEND-9"]}, "a", *args) == [] # a label gone from auto/: flagged, not an error
import tempfile
with tempfile.TemporaryDirectory() as d:
os.makedirs(os.path.join(d, "cases"))
with open(os.path.join(d, "NORDEND.jsonl"), "w") as f:
f.write(json.dumps({"id": "NORDEND-1"}) + "\n{broken\n") # an unreadable label line: no traceback
for c in ({**ok, "neighbours": [], "sources": ["NORDEND-1", "NORDEND-9"]},):
with open(os.path.join(d, "cases", "a.json"), "w") as f:
json.dump(c, f)
with open(os.path.join(d, "cases", ".read.json"), "w") as f:
f.write("{}")
n, errs, flagged = check_cases(d)
assert (n, errs) == (1, []) and flagged == ["cases/a.json source gone from auto/: NORDEND-9"], (n, errs, flagged)
assert bad({"signature": "At NORDEND the targets jump."}) # station name
assert bad({"signature": "A jump of 3 mm."}) # numeric value
assert bad({"stations": ["NORDEND"]}) # unknown field
assert bad({"review": "yesterday"})
label = {"start": "2026-11-04T00:00Z", "end": "2026-11-04T01:00Z", "family": "Perturbation", "description": "x"}
assert not any("unknown family" in e for e in check(label, proposed=True))
print("ok")
def main():
stations = sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl")))
if sys.argv[1:] == ["--check"]:
return self_check()
with_cases = sys.argv[1:] in ([], ["cases"])
stations = [] if sys.argv[1:] == ["cases"] else (
sys.argv[1:] or sorted(os.path.basename(p)[:-6] for p in glob.glob(os.path.join(AUTO, "*.jsonl"))))
errors, flagged, total = [], [], 0
for st in stations:
seen, ids, labels = set(), set(), []
@@ -86,13 +208,22 @@ def main():
if x.get("family") == y.get("family") and x["start"] < y["end"] and y["start"] < x["end"]:
flagged.append(f"{a} and {b}: two {x.get('family')} that overlap")
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
for l in errors:
print(" ERROR", l)
for l in flagged[:40]:
print(" to look at", l)
if len(flagged) > 40:
print(f" … and {len(flagged) - 40} other overlaps")
if stations:
print(f"{total} labels, {len(stations)} stations: {len(errors)} errors, {len(flagged)} overlaps to look at")
for l in errors:
print(" ERROR", l)
for l in flagged[:40]:
print(" to look at", l)
if len(flagged) > 40:
print(f" … and {len(flagged) - 40} other overlaps")
if with_cases:
n, case_errors, gone = check_cases()
print(f"{n} cases: {len(case_errors)} errors, {len(gone)} label sources gone from auto/ to look at (`/cases` removes them)")
for l in case_errors:
print(" ERROR", l)
for l in gone:
print(" to look at", l)
errors += case_errors
sys.exit(1 if errors else 0)