Files
aimsight-detectors/scripts/categories.py
T

243 lines
16 KiB
Python
Raw Normal View History

"""From the detectors' episodes to events, one per family.
One detector per label family (labeling/familles.md): visibility.py, obstruction.py, target_issue.py,
thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its
episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules:
- a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat
shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and
family; otherwise it stays Displacement, cause not established, to be checked on the images (familles.md, "A jump then
the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several
days it crosses (wf-improve session Displacement, mod. 1, S5);
- a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the
curve of a problem target is to be ignored, familles.md);
- a shift that lasts, or too recent to know (SHIFTS), is never absorbed;
- heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of
a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back
starting during it, not the one of the day before coming back within its day (wf-improve session Visibility, mod. 1,
S1: WILDSTRUBEL);
- two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction)
only if they share a sensor (wf-improve session Obstruction, mod. 1: S1, different targets chained into an event of
several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its
bounds (wf-improve session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after
its instant (S3: not the 30 h jump of another target).
Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client"
column of familles.md says; the cause is only given if the variety establishes it, otherwise the images show it.
No detect function: the loader does not take this file for a detector.
Usage: python3 detectors/scripts/categories.py <station folder> (one line per event: family, variety, sensors)
... <folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days
classified alone, with the days of history before it if they exist; only the events touching the chunk are
shown; "nothing" for a quiet chunk)
"""
import functools
import importlib.util
import inspect
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402
# What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134):
# temperature [(t, °C)], hourly median of the components; camera [pose] (pan, tilt, rotation, scale, sigma0);
# metrics {name: [(t, value)]}, all numeric metrics (sensor_CPU_deg_c, sensor_GPU_deg_c, BusVoltage…).
CONTEXT = {"temperature": load_temperature, "camera": load_camera, "metrics": load_metrics}
@functools.lru_cache(maxsize=None)
def _context(folder, key):
return CONTEXT[key](folder)
# The families of labeling/familles.md, in the order of the timeline rows; ignore: greyed zone (None: as its episodes
# say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to
# Diagnosis.
CATEGORIES = {
"Visibility": {"ignore": None, "process": "images"},
"Obstruction": {"ignore": True, "process": "images"},
"Target issue": {"ignore": True, "process": "images"},
"Thermal displacement": {"ignore": False, "process": "none"},
"No data": {"ignore": False, "process": "none"},
"Displacement": {"ignore": False, "process": "check"},
}
JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement
SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed
HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction",
"Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an
# overlapping jump back
MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24)
def detectors():
"""Modules of the *.py next to this file that have detect; a file that does not load is skipped with a message."""
mods = []
for path in sorted(Path(__file__).resolve().parent.glob("*.py")):
if path == Path(__file__).resolve():
continue
try:
spec = importlib.util.spec_from_file_location(f"detector_{path.stem}", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
except Exception as e: # noqa: BLE001 (a broken detector does not block the page)
print(f"Detectors: {path.name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
continue
if hasattr(mod, "detect"):
mods.append(mod)
return mods
def variety(ep):
"""The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of
visibility by level."""
label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize()
return ep["kind"], label, ep.get("level")
def is_thermal(ep):
return ep["kind"] == "Thermal displacement"
def overlap(a, b):
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
def targets(n):
return "one target" if n == 1 else f"{n} targets"
def sentence(group, sensors, references):
"""Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed
jump adds that it has the same cause."""
n, one, ep = len(sensors), len(sensors) == 1, group[0]
jumps = [e for e in group[1:] if e["label"] == JUMP_BACK]
jumped = len({s for e in jumps for s in e["sensors"]})
lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])})
refs = ", including the reference targets" if references & set(sensors) else ""
camera = ep["label"].startswith("Camera")
large = any(e["label"] == "Large shift" for e in group)
sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"]
text = {
"Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. "
"The curves in this period are not reliable: ignore them.",
"Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. "
"The data in this period are less precise: use them with care.",
"Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.",
"Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around "
f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.",
"Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.",
"expansion": ("The camera support expands in the day heat; the curves are corrected for it."
if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.",
"contraction": ("The camera support contracts as it cools down; the curves are corrected for it."
if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.",
JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.",
"shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large
else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting
else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n
else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.")
+ " Check whether the structure moved: look at the images.",
}.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text
if ep["kind"] == "Target issue":
text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target."
if not jumped:
return text
return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1
else f" The jumps of {targets(jumped)} in this period have the same cause.")
def event(group, references):
"""Episodes of the same event, the main one first -> event."""
cat, var = group[0]["kind"], variety(group[0])
c = CATEGORIES.get(cat, {"ignore": None, "process": "none"})
sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"]))
main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps'
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat,
"label": cat, "text": sentence(group, sensors, references),
"ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
"sensors": sensors, "process": c["process"], "episodes": group}
def classify(episodes, references=frozenset()):
"""Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref
sensors, for the sentence."""
eps = sorted(episodes, key=lambda e: e["start"])
# a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene
checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1]
eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)]
# the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several
# targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003)
hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal)
host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h))
and (h["label"] != "Heat shimmer" or h["start"] <= e["start"])
and (h["label"] != "Brief obstruction" or e["start"] == h["start"]
and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None)
for e in eps if e["label"] == JUMP_BACK}
# a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored,
# it takes its family (wf-improve session Displacement, mod. 1, S6)
host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e))
and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h)
and set(e["sensors"]) <= set(h["sensors"])), None))})
groups = {id(e): [e] for e in eps if not host.get(id(e))}
for e in eps:
if host.get(id(e)):
groups[id(host[id(e)])].append(e)
merged = []
for g in groups.values():
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue")
or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None)
end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
prev += g
else:
merged.append(g)
return sorted((event(g, references) for g in merged), key=lambda v: v["start"])
def events(measurements, end=None, folder=None):
"""Measurements (data.load) -> events of all detectors: what the page receives. A detector also receives what its
detect function declares: end, the end of the window read (batch run), for a hole touching it; a CONTEXT key, read
from the station folder (once per folder), empty without a folder."""
episodes = []
for mod in detectors():
try:
params = inspect.signature(mod.detect).parameters
kw = {k: _context(folder, k) if folder else {} if k == "metrics" else [] for k in CONTEXT if k in params}
if "end" in params:
kw["end"] = end
episodes += mod.detect(measurements, **kw)
except Exception as e: # noqa: BLE001
print(f"Detectors: {mod.__name__} skipped ({type(e).__name__}: {e})", file=sys.stderr)
return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"})
if __name__ == "__main__":
import argparse
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
p.add_argument("folder")
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
# workflow as in the real application
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
a = p.parse_args()
measurements = load(a.folder)
ts = [m["t"] for ms in measurements.values() for m in ms]
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
t = start
while t < end:
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
h = t - timedelta(days=a.history)
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else []
if a.chunk:
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
for v in evs:
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
t = b