"""From the detectors' episodes to events, one per family. One detector per label family (labeling/familles.md): visibility.py, obstruction.py, target_issue.py, thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules: - a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and family; otherwise it stays Displacement, cause not established, to be checked on the images (familles.md, "A jump then the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several days it crosses (learner session Displacement, mod. 1, S5); - a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the curve of a problem target is to be ignored, familles.md); - a shift that lasts, or too recent to know (SHIFTS), is never absorbed; - heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back starting during it, not the one of the day before coming back within its day (learner session Visibility, mod. 1, S1: WILDSTRUBEL); - two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction) only if they share a sensor (learner session Obstruction, mod. 1: S1, different targets chained into an event of several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its bounds (learner session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after its instant (S3: not the 30 h jump of another target). Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client" column of familles.md says; the cause is only given if the variety establishes it, otherwise the images show it. No detect function: the loader does not take this file for a detector. Usage: python3 detectors/scripts/categories.py (one line per event: family, variety, sensors) ... [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days classified alone, with the days of history before it if they exist; only the events touching the chunk are shown; "nothing" for a quiet chunk) """ import functools import importlib.util import inspect import os import sys from datetime import datetime, timedelta, timezone from pathlib import Path sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402 # What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134): # temperature [(t, °C)], hourly median of the components; camera [pose] (pan, tilt, rotation, scale, sigma0); # metrics {name: [(t, value)]}, all numeric metrics (sensor_CPU_deg_c, sensor_GPU_deg_c, BusVoltage…). CONTEXT = {"temperature": load_temperature, "camera": load_camera, "metrics": load_metrics} @functools.lru_cache(maxsize=None) def _context(folder, key): return CONTEXT[key](folder) # The families of labeling/familles.md, in the order of the timeline rows; ignore: greyed zone (None: as its episodes # say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to # Diagnosis. CATEGORIES = { "Visibility": {"ignore": None, "process": "images"}, "Obstruction": {"ignore": True, "process": "images"}, "Target issue": {"ignore": True, "process": "images"}, "Thermal displacement": {"ignore": False, "process": "none"}, "No data": {"ignore": False, "process": "none"}, "Displacement": {"ignore": False, "process": "check"}, } JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction", "Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an # overlapping jump back MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24) def detectors(): """Modules of the *.py next to this file that have detect; a file that does not load is skipped with a message.""" mods = [] for path in sorted(Path(__file__).resolve().parent.glob("*.py")): if path == Path(__file__).resolve(): continue try: spec = importlib.util.spec_from_file_location(f"detector_{path.stem}", path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) except Exception as e: # noqa: BLE001 (a broken detector does not block the page) print(f"Detectors: {path.name} skipped ({type(e).__name__}: {e})", file=sys.stderr) continue if hasattr(mod, "detect"): mods.append(mod) return mods def variety(ep): """The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of visibility by level.""" label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize() return ep["kind"], label, ep.get("level") def is_thermal(ep): return ep["kind"] == "Thermal displacement" def overlap(a, b): return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order def targets(n): return "one target" if n == 1 else f"{n} targets" def sentence(group, sensors, references): """Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed jump adds that it has the same cause.""" n, one, ep = len(sensors), len(sensors) == 1, group[0] jumps = [e for e in group[1:] if e["label"] == JUMP_BACK] jumped = len({s for e in jumps for s in e["sensors"]}) lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])}) refs = ", including the reference targets" if references & set(sensors) else "" camera = ep["label"].startswith("Camera") large = any(e["label"] == "Large shift" for e in group) sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"] text = { "Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. " "The curves in this period are not reliable: ignore them.", "Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. " "The data in this period are less precise: use them with care.", "Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.", "Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around " f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.", "Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.", "expansion": ("The camera support expands in the day heat; the curves are corrected for it." if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.", "contraction": ("The camera support contracts as it cools down; the curves are corrected for it." if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.", JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.", "shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.") + " Check whether the structure moved: look at the images.", }.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text if ep["kind"] == "Target issue": text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target." if not jumped: return text return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1 else f" The jumps of {targets(jumped)} in this period have the same cause.") def event(group, references): """Episodes of the same event, the main one first -> event.""" cat, var = group[0]["kind"], variety(group[0]) c = CATEGORIES.get(cat, {"ignore": None, "process": "none"}) sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"])) main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps' return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat, "label": cat, "text": sentence(group, sensors, references), "ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main), "sensors": sensors, "process": c["process"], "episodes": group} def classify(episodes, references=frozenset()): """Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref sensors, for the sentence.""" eps = sorted(episodes, key=lambda e: e["start"]) # a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1] eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)] # the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several # targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003) hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal) host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h)) and (h["label"] != "Heat shimmer" or h["start"] <= e["start"]) and (h["label"] != "Brief obstruction" or e["start"] == h["start"] and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None) for e in eps if e["label"] == JUMP_BACK} # a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored, # it takes its family (learner session Displacement, mod. 1, S6) host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e)) and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h) and set(e["sensors"]) <= set(h["sensors"])), None))}) groups = {id(e): [e] for e in eps if not host.get(id(e))} for e in eps: if host.get(id(e)): groups[id(host[id(e)])].append(e) merged = [] for g in groups.values(): prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue") or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None) end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE: prev += g else: merged.append(g) return sorted((event(g, references) for g in merged), key=lambda v: v["start"]) def events(measurements, end=None, folder=None): """Measurements (data.load) -> events of all detectors: what the page receives. A detector also receives what its detect function declares: end, the end of the window read (batch run), for a hole touching it; a CONTEXT key, read from the station folder (once per folder), empty without a folder.""" episodes = [] for mod in detectors(): try: params = inspect.signature(mod.detect).parameters kw = {k: _context(folder, k) if folder else {} if k == "metrics" else [] for k in CONTEXT if k in params} if "end" in params: kw["end"] = end episodes += mod.detect(measurements, **kw) except Exception as e: # noqa: BLE001 print(f"Detectors: {mod.__name__} skipped ({type(e).__name__}: {e})", file=sys.stderr) return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"}) if __name__ == "__main__": import argparse p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.") p.add_argument("folder") p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)") p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)") p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)") # without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30: # absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the # learner as in the real application p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)") a = p.parse_args() measurements = load(a.folder) ts = [m["t"] for ms in measurements.values() for m in ms] day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731 start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0) end = day(a.end) if a.end else max(ts) + timedelta(seconds=1) t = start while t < end: b = min(t + timedelta(days=a.chunk), end) if a.chunk else end h = t - timedelta(days=a.history) chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])} evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else [] if a.chunk: print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}") for v in evs: print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept", f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join( f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—") t = b