From 4188efe20122e566598fb287145053c76597635b Mon Sep 17 00:00:00 2001 From: maxime Date: Mon, 16 Nov 2026 17:28:04 +0100 Subject: [PATCH] categories: plain absorption and merge rules, the event text is its first episode's A jump back joins an overlapping Visibility episode, or an Obstruction or Target issue episode holding all its targets; a slow drift, a Target issue episode holding its targets. Removed (their benchmark contribution, each alone): heat shimmer dropped next to a shift of several targets (Visibility found +1), HOSTS with the thermal wave (none), the special cases of a Brief obstruction, of heat shimmer and of the thermal wave as hosts (one jump back at WILDSTRUBEL, already lost), SHIFTS, variety() reduced to (label, level). sentence() removed: each detector writes the text of its episodes; the event adds one sentence for the jumps it absorbed. events() no longer takes anything from the folder. Benchmark Totals (from the previous commit): Visibility found 54 -> 55 %, correct 71 -> 72 %; the other families unchanged. regression.py now runs in about 1 min (was 3.5). Co-Authored-By: Claude Opus 5.5 --- scripts/categories.py | 182 ++++++++++++------------------------------ 1 file changed, 51 insertions(+), 131 deletions(-) diff --git a/scripts/categories.py b/scripts/categories.py index 667ccfa..79bc7f5 100644 --- a/scripts/categories.py +++ b/scripts/categories.py @@ -1,32 +1,19 @@ -"""From the detectors' episodes to events, one per family. +"""From the detectors' episodes to events, one family each. -One detector per label family (the families document): visibility.py, obstruction.py, target_issue.py, -thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its -episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules: -- a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat - shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and - family; otherwise it stays Displacement, cause not established, to be checked on the images (the families document, "A jump then - the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several - days it crosses (wf-improve session Displacement, mod. 1, S5); -- a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the - curve of a problem target is to be ignored, the families document); -- a shift that lasts, or too recent to know (SHIFTS), is never absorbed; -- heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of - a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back - starting during it, not the one of the day before coming back within its day (wf-improve session Visibility, mod. 1, - S1: WILDSTRUBEL); -- two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction) - only if they share a sensor (wf-improve session Obstruction, mod. 1: S1, different targets chained into an event of - several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its - bounds (wf-improve session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after - its instant (S3: not the 30 h jump of another target). -Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client" -column of the families document says; the cause is only given if the variety establishes it, otherwise the images show it. -No detect function: the loader does not take this file for a detector. -Usage: python3 scripts/categories.py (one line per event: family, variety, sensors) - ... [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days - classified alone, with the days of history before it if they exist; only the events touching the chunk are - shown; "nothing" for a quiet chunk) +Each module of DETECTORS (next to this file) returns episodes: start, end, kind (the family), label (the variety), text, +ignore, sensors (a Poor visibility also has its level). classify() makes them events: +- a target already in an overlapping Target issue episode is dropped from an Obstruction episode: its curve is already + to be ignored; +- a jump back to normal (JUMP_BACK) overlapping a Visibility episode joins its event: rain, fog or heat shimmer + explains it; otherwise it stays Displacement, its cause to be checked on the images; +- a jump back whose targets are all in an overlapping Obstruction or Target issue episode joins it (something in front + of them explains it, or the curve of a problem target is to be ignored); a slow drift, in a Target issue episode; +- episodes of the same variety less than MERGE apart make one event (validated rule, 2026-10-24); an Obstruction or a + Target issue only with an episode of the same target (wf-improve Obstruction, mod. 1, S1; Target issue, mod. 1, S3). +The text of an event is that of its first episode, plus a sentence for the jumps it absorbed. +Usage: python3 scripts/categories.py [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk N] [--history N] + one line per event; --chunk: each chunk of N days classified alone, with the days of history before it; only the + events touching the chunk are shown, "nothing" for a quiet chunk. """ import importlib import sys @@ -37,14 +24,13 @@ from pathlib import Path # sibling of this one (one copy of each repository in the home directory). HERE = Path(__file__).resolve().parent sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts")] +from common import targets # noqa: E402 +from displacement import JUMP_BACK # noqa: E402 from data import iso, load # noqa: E402 -DETECTORS = ("displacement", "no_data", "target_issue", "thermal_displacement", "visibility") # modules next to this file - - -# The families, in the order of the timeline rows; ignore: greyed zone (None: as its episodes -# say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to -# Diagnosis. +DETECTORS = ("displacement", "no_data", "target_issue", "thermal_displacement", "visibility") +# The families, in the order of the timeline rows; ignore: greyed zone (None: as its episodes say); process: Images +# button, nothing, or Images and a link to Diagnosis. CATEGORIES = { "Visibility": {"ignore": None, "process": "images"}, "Obstruction": {"ignore": True, "process": "images"}, @@ -53,138 +39,72 @@ CATEGORIES = { "No data": {"ignore": False, "process": "none"}, "Displacement": {"ignore": False, "process": "check"}, } -JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement -SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed -HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction", - "Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an - # overlapping jump back -MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24) +MERGE = timedelta(hours=1) def variety(ep): - """The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of - visibility by level.""" - label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize() - return ep["kind"], label, ep.get("level") - - -def is_thermal(ep): - return ep["kind"] == "Thermal displacement" + return ep["label"], ep.get("level") def overlap(a, b): return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order -def targets(n): - return "one target" if n == 1 else f"{n} targets" +def host(e, eps): + """The episode whose event a jump back or a slow drift joins, or None.""" + kinds = {JUMP_BACK: ("Obstruction", "Target issue"), "Slow drift": ("Target issue",)}.get(e["label"], ()) + return next((h for h in eps if overlap(e, h) and (h["kind"] == "Visibility" and e["label"] == JUMP_BACK + or h["kind"] in kinds and set(e["sensors"]) <= set(h["sensors"]))), None) -def sentence(group, sensors, references): - """Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed - jump adds that it has the same cause.""" - n, one, ep = len(sensors), len(sensors) == 1, group[0] - jumps = [e for e in group[1:] if e["label"] == JUMP_BACK] - jumped = len({s for e in jumps for s in e["sensors"]}) - lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])}) - refs = ", including the reference targets" if references & set(sensors) else "" - camera = ep["label"].startswith("Camera") - large = any(e["label"] == "Large shift" for e in group) - sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"] - text = { - "Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. " - "The curves in this period are not reliable: ignore them.", - "Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. " - "The data in this period are less precise: use them with care.", - "Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.", - "Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around " - f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.", - "Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.", - "expansion": ("The camera support expands in the day heat; the curves are corrected for it." - if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.", - "contraction": ("The camera support contracts as it cools down; the curves are corrected for it." - if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.", - JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.", - "shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large - else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting - else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n - else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.") - + " Check whether the structure moved: look at the images.", - }.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text - if ep["kind"] == "Target issue": - text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target." - if not jumped: - return text - return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1 - else f" The jumps of {targets(jumped)} in this period have the same cause.") +def event(group): + """Episodes of an event, the main one first -> event.""" + cat, c = group[0]["kind"], CATEGORIES[group[0]["kind"]] + main = [e for e in group if variety(e) == variety(group[0])] # the duration is theirs, not the absorbed jumps' + jumped = len({s for e in group if e["label"] == JUMP_BACK and e not in main for s in e["sensors"]}) + text = group[0]["text"] + (f" The jumps of {targets(jumped)} in this period have the same cause." if jumped else "") + return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat, "label": cat, + "text": text, "ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main), + "sensors": [] if any(not e["sensors"] for e in group) else sorted({s for e in group for s in e["sensors"]}), + "process": c["process"], "episodes": group} -def event(group, references): - """Episodes of the same event, the main one first -> event.""" - cat, var = group[0]["kind"], variety(group[0]) - c = CATEGORIES.get(cat, {"ignore": None, "process": "none"}) - sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"])) - main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps' - return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat, - "label": cat, "text": sentence(group, sensors, references), - "ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main), - "sensors": sensors, "process": c["process"], "episodes": group} - - -def classify(episodes, references=frozenset()): - """Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref - sensors, for the sentence.""" +def classify(episodes): + """Episodes of all detectors -> events sorted by start (rules: module docstring).""" eps = sorted(episodes, key=lambda e: e["start"]) - # a sensor already in an overlapping Target issue episode is not Obstruction: its curve is already to be ignored issues = [e for e in eps if e["kind"] == "Target issue"] for e in eps: if e["kind"] == "Obstruction": e["sensors"] = [s for s in e["sensors"] if not any(s in h["sensors"] and overlap(e, h) for h in issues)] eps = [e for e in eps if e["kind"] != "Obstruction" or e["sensors"]] - # a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene - checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1] - eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)] - # the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several - # targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003) - hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal) - host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h)) - and (h["label"] != "Heat shimmer" or h["start"] <= e["start"]) - and (h["label"] != "Brief obstruction" or e["start"] == h["start"] - and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None) - for e in eps if e["label"] == JUMP_BACK} - # a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored, - # it takes its family (wf-improve session Displacement, mod. 1, S6) - host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e)) - and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h) - and set(e["sensors"]) <= set(h["sensors"])), None))}) - groups = {id(e): [e] for e in eps if not host.get(id(e))} + hosts = {id(e): h for e in eps if (h := host(e, eps))} + groups = {id(e): [e] for e in eps if id(e) not in hosts} for e in eps: - if host.get(id(e)): - groups[id(host[id(e)])].append(e) + if id(e) in hosts: + groups[id(hosts[id(e)])].append(e) merged = [] for g in groups.values(): - prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue") - or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None) - end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None + prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ( + "Obstruction", "Target issue") or set(g[0]["sensors"]) & set(m[0]["sensors"]))), None) # the same target + end = prev and max(e["end"] for e in prev if variety(e) == variety(g[0])) if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE: prev += g else: merged.append(g) - return sorted((event(g, references) for g in merged), key=lambda v: v["start"]) + return sorted((event(g) for g in merged), key=lambda v: v["start"]) def events(measurements, end=None, folder=None): """Measurements (data.load) -> events of all detectors: what the page receives. end: the end of the window read - (batch run), for a hole touching it; folder: unused, kept for the callers. A detector - that fails is skipped with a message (a broken detector does not block the page).""" + (batch run), for a hole touching it; folder: unused, kept for the callers. A detector that fails is skipped with a + message (a broken detector does not block the page).""" episodes = [] for name in DETECTORS: try: - kw = {"end": end} if name == "no_data" else {} - episodes += importlib.import_module(name).detect(measurements, **kw) + episodes += importlib.import_module(name).detect(measurements, **({"end": end} if name == "no_data" else {})) except Exception as e: # noqa: BLE001 print(f"Detectors: {name} skipped ({type(e).__name__}: {e})", file=sys.stderr) - return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"}) + return classify(episodes) if __name__ == "__main__":