"""From the detectors' episodes to events, one family each. Each module of DETECTORS (next to this file) returns episodes: start, end, kind (the family), label (the variety), text, ignore, sensors (a Poor visibility also has its level). classify() makes them events: - a target already in an overlapping Target issue episode is dropped from an Obstruction episode: its curve is already to be ignored; - a jump back to normal (JUMP_BACK) overlapping a Visibility episode joins its event: rain, fog or heat shimmer explains it; otherwise it stays Displacement, its cause to be checked on the images; - a jump back whose targets are all in an overlapping Obstruction or Target issue episode joins it (something in front of them explains it, or the curve of a problem target is to be ignored); a slow drift, in a Target issue episode; - episodes of the same variety less than MERGE apart make one event (validated rule, 2026-10-24); an Obstruction or a Target issue only with an episode of the same target (wf-improve Obstruction, mod. 1, S1; Target issue, mod. 1, S3). The text of an event is that of its first episode, plus a sentence for the jumps it absorbed. Usage: python3 scripts/categories.py [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk N] [--history N] one line per event; --chunk: each chunk of N days classified alone, with the days of history before it; only the events touching the chunk are shown, "nothing" for a quiet chunk. """ import importlib import sys from datetime import datetime, timedelta, timezone from pathlib import Path # The detectors next to this file, and the single reader of the measurements: data.py of the tools repository, the # sibling of this one (one copy of each repository in the home directory). HERE = Path(__file__).resolve().parent sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts")] from common import targets # noqa: E402 from displacement import JUMP_BACK # noqa: E402 from data import iso, load # noqa: E402 DETECTORS = ("displacement", "no_data", "target_issue", "thermal_displacement", "visibility") # The families, in the order of the timeline rows; ignore: greyed zone (None: as its episodes say); process: Images # button, nothing, or Images and a link to Diagnosis. CATEGORIES = { "Visibility": {"ignore": None, "process": "images"}, "Obstruction": {"ignore": True, "process": "images"}, "Target issue": {"ignore": True, "process": "images"}, "Thermal displacement": {"ignore": False, "process": "none"}, "No data": {"ignore": False, "process": "none"}, "Displacement": {"ignore": False, "process": "check"}, } MERGE = timedelta(hours=1) def variety(ep): return ep["label"], ep.get("level") def overlap(a, b): return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order def host(e, eps): """The episode whose event a jump back or a slow drift joins, or None.""" kinds = {JUMP_BACK: ("Obstruction", "Target issue"), "Slow drift": ("Target issue",)}.get(e["label"], ()) return next((h for h in eps if overlap(e, h) and (h["kind"] == "Visibility" and e["label"] == JUMP_BACK or h["kind"] in kinds and set(e["sensors"]) <= set(h["sensors"]))), None) def event(group): """Episodes of an event, the main one first -> event.""" cat, c = group[0]["kind"], CATEGORIES[group[0]["kind"]] main = [e for e in group if variety(e) == variety(group[0])] # the duration is theirs, not the absorbed jumps' jumped = len({s for e in group if e["label"] == JUMP_BACK and e not in main for s in e["sensors"]}) text = group[0]["text"] + (f" The jumps of {targets(jumped)} in this period have the same cause." if jumped else "") return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat, "label": cat, "text": text, "ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main), "sensors": [] if any(not e["sensors"] for e in group) else sorted({s for e in group for s in e["sensors"]}), "process": c["process"], "episodes": group} def classify(episodes): """Episodes of all detectors -> events sorted by start (rules: module docstring).""" eps = sorted(episodes, key=lambda e: e["start"]) issues = [e for e in eps if e["kind"] == "Target issue"] for e in eps: if e["kind"] == "Obstruction": e["sensors"] = [s for s in e["sensors"] if not any(s in h["sensors"] and overlap(e, h) for h in issues)] eps = [e for e in eps if e["kind"] != "Obstruction" or e["sensors"]] hosts = {id(e): h for e in eps if (h := host(e, eps))} groups = {id(e): [e] for e in eps if id(e) not in hosts} for e in eps: if id(e) in hosts: groups[id(hosts[id(e)])].append(e) merged = [] for g in groups.values(): prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ( "Obstruction", "Target issue") or set(g[0]["sensors"]) & set(m[0]["sensors"]))), None) # the same target end = prev and max(e["end"] for e in prev if variety(e) == variety(g[0])) if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE: prev += g else: merged.append(g) return sorted((event(g) for g in merged), key=lambda v: v["start"]) def events(measurements, end=None, folder=None): """Measurements (data.load) -> events of all detectors: what the page receives. end: the end of the window read (batch run), for a hole touching it; folder: unused, kept for the callers. A detector that fails is skipped with a message (a broken detector does not block the page).""" episodes = [] for name in DETECTORS: try: episodes += importlib.import_module(name).detect(measurements, **({"end": end} if name == "no_data" else {})) except Exception as e: # noqa: BLE001 print(f"Detectors: {name} skipped ({type(e).__name__}: {e})", file=sys.stderr) return classify(episodes) if __name__ == "__main__": import argparse p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.") p.add_argument("folder") p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)") p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)") p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)") # without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30: # absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the # workflow as in the real application p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)") a = p.parse_args() measurements = load(a.folder) ts = [m["t"] for ms in measurements.values() for m in ms] day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731 start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0) end = day(a.end) if a.end else max(ts) + timedelta(seconds=1) t = start while t < end: b = min(t + timedelta(days=a.chunk), end) if a.chunk else end h = t - timedelta(days=a.history) chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])} evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else [] if a.chunk: print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}") for v in evs: print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept", f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join( f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—") t = b