2026-11-13 12:19:52 +01:00
|
|
|
"""From the detectors' episodes to events, one per family.
|
|
|
|
|
|
|
|
|
|
One detector per label family (labeling/familles.md): visibility.py, obstruction.py, target_issue.py,
|
|
|
|
|
thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its
|
|
|
|
|
episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules:
|
|
|
|
|
- a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat
|
|
|
|
|
shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and
|
|
|
|
|
family; otherwise it stays Displacement, cause not established, to be checked on the images (familles.md, "A jump then
|
|
|
|
|
the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several
|
2026-11-13 12:26:02 +01:00
|
|
|
days it crosses (wf-improve session Displacement, mod. 1, S5);
|
2026-11-13 12:19:52 +01:00
|
|
|
- a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the
|
|
|
|
|
curve of a problem target is to be ignored, familles.md);
|
|
|
|
|
- a shift that lasts, or too recent to know (SHIFTS), is never absorbed;
|
|
|
|
|
- heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of
|
|
|
|
|
a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back
|
2026-11-13 12:26:02 +01:00
|
|
|
starting during it, not the one of the day before coming back within its day (wf-improve session Visibility, mod. 1,
|
2026-11-13 12:19:52 +01:00
|
|
|
S1: WILDSTRUBEL);
|
|
|
|
|
- two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction)
|
2026-11-13 12:26:02 +01:00
|
|
|
only if they share a sensor (wf-improve session Obstruction, mod. 1: S1, different targets chained into an event of
|
2026-11-13 12:19:52 +01:00
|
|
|
several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its
|
2026-11-13 12:26:02 +01:00
|
|
|
bounds (wf-improve session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after
|
2026-11-13 12:19:52 +01:00
|
|
|
its instant (S3: not the 30 h jump of another target).
|
|
|
|
|
Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client"
|
|
|
|
|
column of familles.md says; the cause is only given if the variety establishes it, otherwise the images show it.
|
|
|
|
|
No detect function: the loader does not take this file for a detector.
|
|
|
|
|
Usage: python3 detectors/scripts/categories.py <station folder> (one line per event: family, variety, sensors)
|
|
|
|
|
... <folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days
|
|
|
|
|
classified alone, with the days of history before it if they exist; only the events touching the chunk are
|
|
|
|
|
shown; "nothing" for a quiet chunk)
|
|
|
|
|
"""
|
|
|
|
|
import functools
|
|
|
|
|
import importlib.util
|
|
|
|
|
import inspect
|
|
|
|
|
import os
|
|
|
|
|
import sys
|
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
|
|
|
|
|
from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402
|
|
|
|
|
|
|
|
|
|
# What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134):
|
|
|
|
|
# temperature [(t, °C)], hourly median of the components; camera [pose] (pan, tilt, rotation, scale, sigma0);
|
|
|
|
|
# metrics {name: [(t, value)]}, all numeric metrics (sensor_CPU_deg_c, sensor_GPU_deg_c, BusVoltage…).
|
|
|
|
|
CONTEXT = {"temperature": load_temperature, "camera": load_camera, "metrics": load_metrics}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
|
|
|
def _context(folder, key):
|
|
|
|
|
return CONTEXT[key](folder)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# The families of labeling/familles.md, in the order of the timeline rows; ignore: greyed zone (None: as its episodes
|
|
|
|
|
# say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to
|
|
|
|
|
# Diagnosis.
|
|
|
|
|
CATEGORIES = {
|
|
|
|
|
"Visibility": {"ignore": None, "process": "images"},
|
|
|
|
|
"Obstruction": {"ignore": True, "process": "images"},
|
|
|
|
|
"Target issue": {"ignore": True, "process": "images"},
|
|
|
|
|
"Thermal displacement": {"ignore": False, "process": "none"},
|
|
|
|
|
"No data": {"ignore": False, "process": "none"},
|
|
|
|
|
"Displacement": {"ignore": False, "process": "check"},
|
|
|
|
|
}
|
|
|
|
|
JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement
|
|
|
|
|
SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed
|
|
|
|
|
HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction",
|
|
|
|
|
"Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an
|
|
|
|
|
# overlapping jump back
|
|
|
|
|
MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def detectors():
|
|
|
|
|
"""Modules of the *.py next to this file that have detect; a file that does not load is skipped with a message."""
|
|
|
|
|
mods = []
|
|
|
|
|
for path in sorted(Path(__file__).resolve().parent.glob("*.py")):
|
|
|
|
|
if path == Path(__file__).resolve():
|
|
|
|
|
continue
|
|
|
|
|
try:
|
|
|
|
|
spec = importlib.util.spec_from_file_location(f"detector_{path.stem}", path)
|
|
|
|
|
mod = importlib.util.module_from_spec(spec)
|
|
|
|
|
spec.loader.exec_module(mod)
|
|
|
|
|
except Exception as e: # noqa: BLE001 (a broken detector does not block the page)
|
|
|
|
|
print(f"Detectors: {path.name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
|
|
|
|
|
continue
|
|
|
|
|
if hasattr(mod, "detect"):
|
|
|
|
|
mods.append(mod)
|
|
|
|
|
return mods
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def variety(ep):
|
|
|
|
|
"""The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of
|
|
|
|
|
visibility by level."""
|
|
|
|
|
label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize()
|
|
|
|
|
return ep["kind"], label, ep.get("level")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_thermal(ep):
|
|
|
|
|
return ep["kind"] == "Thermal displacement"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def overlap(a, b):
|
|
|
|
|
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def targets(n):
|
|
|
|
|
return "one target" if n == 1 else f"{n} targets"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sentence(group, sensors, references):
|
|
|
|
|
"""Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed
|
|
|
|
|
jump adds that it has the same cause."""
|
|
|
|
|
n, one, ep = len(sensors), len(sensors) == 1, group[0]
|
|
|
|
|
jumps = [e for e in group[1:] if e["label"] == JUMP_BACK]
|
|
|
|
|
jumped = len({s for e in jumps for s in e["sensors"]})
|
|
|
|
|
lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])})
|
|
|
|
|
refs = ", including the reference targets" if references & set(sensors) else ""
|
|
|
|
|
camera = ep["label"].startswith("Camera")
|
|
|
|
|
large = any(e["label"] == "Large shift" for e in group)
|
|
|
|
|
sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"]
|
|
|
|
|
text = {
|
|
|
|
|
"Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. "
|
|
|
|
|
"The curves in this period are not reliable: ignore them.",
|
|
|
|
|
"Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. "
|
|
|
|
|
"The data in this period are less precise: use them with care.",
|
|
|
|
|
"Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.",
|
|
|
|
|
"Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around "
|
|
|
|
|
f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.",
|
|
|
|
|
"Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.",
|
|
|
|
|
"expansion": ("The camera support expands in the day heat; the curves are corrected for it."
|
|
|
|
|
if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.",
|
|
|
|
|
"contraction": ("The camera support contracts as it cools down; the curves are corrected for it."
|
|
|
|
|
if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.",
|
|
|
|
|
JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.",
|
|
|
|
|
"shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large
|
|
|
|
|
else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting
|
|
|
|
|
else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n
|
|
|
|
|
else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.")
|
|
|
|
|
+ " Check whether the structure moved: look at the images.",
|
|
|
|
|
}.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text
|
|
|
|
|
if ep["kind"] == "Target issue":
|
|
|
|
|
text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target."
|
|
|
|
|
if not jumped:
|
|
|
|
|
return text
|
|
|
|
|
return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1
|
|
|
|
|
else f" The jumps of {targets(jumped)} in this period have the same cause.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def event(group, references):
|
|
|
|
|
"""Episodes of the same event, the main one first -> event."""
|
|
|
|
|
cat, var = group[0]["kind"], variety(group[0])
|
|
|
|
|
c = CATEGORIES.get(cat, {"ignore": None, "process": "none"})
|
|
|
|
|
sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"]))
|
|
|
|
|
main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps'
|
|
|
|
|
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat,
|
|
|
|
|
"label": cat, "text": sentence(group, sensors, references),
|
|
|
|
|
"ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
|
|
|
|
|
"sensors": sensors, "process": c["process"], "episodes": group}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def classify(episodes, references=frozenset()):
|
|
|
|
|
"""Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref
|
|
|
|
|
sensors, for the sentence."""
|
|
|
|
|
eps = sorted(episodes, key=lambda e: e["start"])
|
|
|
|
|
# a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene
|
|
|
|
|
checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1]
|
|
|
|
|
eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)]
|
|
|
|
|
# the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several
|
|
|
|
|
# targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003)
|
|
|
|
|
hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal)
|
|
|
|
|
host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h))
|
|
|
|
|
and (h["label"] != "Heat shimmer" or h["start"] <= e["start"])
|
|
|
|
|
and (h["label"] != "Brief obstruction" or e["start"] == h["start"]
|
|
|
|
|
and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None)
|
|
|
|
|
for e in eps if e["label"] == JUMP_BACK}
|
|
|
|
|
# a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored,
|
2026-11-13 12:26:02 +01:00
|
|
|
# it takes its family (wf-improve session Displacement, mod. 1, S6)
|
2026-11-13 12:19:52 +01:00
|
|
|
host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e))
|
|
|
|
|
and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h)
|
|
|
|
|
and set(e["sensors"]) <= set(h["sensors"])), None))})
|
|
|
|
|
groups = {id(e): [e] for e in eps if not host.get(id(e))}
|
|
|
|
|
for e in eps:
|
|
|
|
|
if host.get(id(e)):
|
|
|
|
|
groups[id(host[id(e)])].append(e)
|
|
|
|
|
merged = []
|
|
|
|
|
for g in groups.values():
|
|
|
|
|
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue")
|
|
|
|
|
or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None)
|
|
|
|
|
end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None
|
|
|
|
|
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
|
|
|
|
|
prev += g
|
|
|
|
|
else:
|
|
|
|
|
merged.append(g)
|
|
|
|
|
return sorted((event(g, references) for g in merged), key=lambda v: v["start"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def events(measurements, end=None, folder=None):
|
|
|
|
|
"""Measurements (data.load) -> events of all detectors: what the page receives. A detector also receives what its
|
|
|
|
|
detect function declares: end, the end of the window read (batch run), for a hole touching it; a CONTEXT key, read
|
|
|
|
|
from the station folder (once per folder), empty without a folder."""
|
|
|
|
|
episodes = []
|
|
|
|
|
for mod in detectors():
|
|
|
|
|
try:
|
|
|
|
|
params = inspect.signature(mod.detect).parameters
|
|
|
|
|
kw = {k: _context(folder, k) if folder else {} if k == "metrics" else [] for k in CONTEXT if k in params}
|
|
|
|
|
if "end" in params:
|
|
|
|
|
kw["end"] = end
|
|
|
|
|
episodes += mod.detect(measurements, **kw)
|
|
|
|
|
except Exception as e: # noqa: BLE001
|
|
|
|
|
print(f"Detectors: {mod.__name__} skipped ({type(e).__name__}: {e})", file=sys.stderr)
|
|
|
|
|
return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
import argparse
|
|
|
|
|
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
|
|
|
|
|
p.add_argument("folder")
|
|
|
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
|
|
|
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
|
|
|
|
|
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
|
|
|
|
|
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
|
|
|
|
|
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
|
2026-11-13 12:26:02 +01:00
|
|
|
# workflow as in the real application
|
2026-11-13 12:19:52 +01:00
|
|
|
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
|
|
|
|
|
a = p.parse_args()
|
|
|
|
|
measurements = load(a.folder)
|
|
|
|
|
ts = [m["t"] for ms in measurements.values() for m in ms]
|
|
|
|
|
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
|
|
|
|
|
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
|
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
|
|
|
|
|
t = start
|
|
|
|
|
while t < end:
|
|
|
|
|
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
|
|
|
|
|
h = t - timedelta(days=a.history)
|
|
|
|
|
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
|
|
|
|
|
evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else []
|
|
|
|
|
if a.chunk:
|
|
|
|
|
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
|
|
|
|
|
for v in evs:
|
|
|
|
|
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
|
|
|
|
|
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
|
|
|
|
|
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
|
|
|
|
|
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
|
|
|
|
|
t = b
|