Each module returns every family it finds (kind); categories.py imports DETECTORS by name (no introspection, no CONTEXT) and drops from Obstruction the sensors already in an overlapping Target issue episode (formerly obstruction.py). No detector computed twice. Benchmark identical to the baseline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
228 lines
15 KiB
Python
228 lines
15 KiB
Python
"""From the detectors' episodes to events, one per family.
|
|
|
|
One detector per label family (the families document): visibility.py, obstruction.py, target_issue.py,
|
|
thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its
|
|
episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules:
|
|
- a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat
|
|
shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and
|
|
family; otherwise it stays Displacement, cause not established, to be checked on the images (the families document, "A jump then
|
|
the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several
|
|
days it crosses (wf-improve session Displacement, mod. 1, S5);
|
|
- a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the
|
|
curve of a problem target is to be ignored, the families document);
|
|
- a shift that lasts, or too recent to know (SHIFTS), is never absorbed;
|
|
- heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of
|
|
a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back
|
|
starting during it, not the one of the day before coming back within its day (wf-improve session Visibility, mod. 1,
|
|
S1: WILDSTRUBEL);
|
|
- two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction)
|
|
only if they share a sensor (wf-improve session Obstruction, mod. 1: S1, different targets chained into an event of
|
|
several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its
|
|
bounds (wf-improve session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after
|
|
its instant (S3: not the 30 h jump of another target).
|
|
Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client"
|
|
column of the families document says; the cause is only given if the variety establishes it, otherwise the images show it.
|
|
No detect function: the loader does not take this file for a detector.
|
|
Usage: python3 scripts/categories.py <station folder> (one line per event: family, variety, sensors)
|
|
... <folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days
|
|
classified alone, with the days of history before it if they exist; only the events touching the chunk are
|
|
shown; "nothing" for a quiet chunk)
|
|
"""
|
|
import functools
|
|
import importlib
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
# The detectors next to this file, and the single reader of the measurements: data.py of the tools repository, the
|
|
# sibling of this one (one copy of each repository in the home directory).
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts")]
|
|
from data import iso, load, load_camera, load_temperature # noqa: E402
|
|
|
|
DETECTORS = ("displacement", "no_data", "target_issue", "thermal_displacement", "visibility") # modules next to this file
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _station(folder):
|
|
"""What thermal_displacement reads besides the measurements (D-124): temperature, camera pose; once per folder."""
|
|
return {"temperature": load_temperature(folder), "camera": load_camera(folder)} if folder else {}
|
|
|
|
|
|
# The families, in the order of the timeline rows; ignore: greyed zone (None: as its episodes
|
|
# say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to
|
|
# Diagnosis.
|
|
CATEGORIES = {
|
|
"Visibility": {"ignore": None, "process": "images"},
|
|
"Obstruction": {"ignore": True, "process": "images"},
|
|
"Target issue": {"ignore": True, "process": "images"},
|
|
"Thermal displacement": {"ignore": False, "process": "none"},
|
|
"No data": {"ignore": False, "process": "none"},
|
|
"Displacement": {"ignore": False, "process": "check"},
|
|
}
|
|
JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement
|
|
SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed
|
|
HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction",
|
|
"Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an
|
|
# overlapping jump back
|
|
MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24)
|
|
|
|
|
|
def variety(ep):
|
|
"""The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of
|
|
visibility by level."""
|
|
label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize()
|
|
return ep["kind"], label, ep.get("level")
|
|
|
|
|
|
def is_thermal(ep):
|
|
return ep["kind"] == "Thermal displacement"
|
|
|
|
|
|
def overlap(a, b):
|
|
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
|
|
|
|
|
|
def targets(n):
|
|
return "one target" if n == 1 else f"{n} targets"
|
|
|
|
|
|
def sentence(group, sensors, references):
|
|
"""Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed
|
|
jump adds that it has the same cause."""
|
|
n, one, ep = len(sensors), len(sensors) == 1, group[0]
|
|
jumps = [e for e in group[1:] if e["label"] == JUMP_BACK]
|
|
jumped = len({s for e in jumps for s in e["sensors"]})
|
|
lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])})
|
|
refs = ", including the reference targets" if references & set(sensors) else ""
|
|
camera = ep["label"].startswith("Camera")
|
|
large = any(e["label"] == "Large shift" for e in group)
|
|
sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"]
|
|
text = {
|
|
"Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. "
|
|
"The curves in this period are not reliable: ignore them.",
|
|
"Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. "
|
|
"The data in this period are less precise: use them with care.",
|
|
"Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.",
|
|
"Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around "
|
|
f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.",
|
|
"Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.",
|
|
"expansion": ("The camera support expands in the day heat; the curves are corrected for it."
|
|
if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.",
|
|
"contraction": ("The camera support contracts as it cools down; the curves are corrected for it."
|
|
if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.",
|
|
JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.",
|
|
"shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large
|
|
else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting
|
|
else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n
|
|
else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.")
|
|
+ " Check whether the structure moved: look at the images.",
|
|
}.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text
|
|
if ep["kind"] == "Target issue":
|
|
text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target."
|
|
if not jumped:
|
|
return text
|
|
return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1
|
|
else f" The jumps of {targets(jumped)} in this period have the same cause.")
|
|
|
|
|
|
def event(group, references):
|
|
"""Episodes of the same event, the main one first -> event."""
|
|
cat, var = group[0]["kind"], variety(group[0])
|
|
c = CATEGORIES.get(cat, {"ignore": None, "process": "none"})
|
|
sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"]))
|
|
main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps'
|
|
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat,
|
|
"label": cat, "text": sentence(group, sensors, references),
|
|
"ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
|
|
"sensors": sensors, "process": c["process"], "episodes": group}
|
|
|
|
|
|
def classify(episodes, references=frozenset()):
|
|
"""Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref
|
|
sensors, for the sentence."""
|
|
eps = sorted(episodes, key=lambda e: e["start"])
|
|
# a sensor already in an overlapping Target issue episode is not Obstruction: its curve is already to be ignored
|
|
issues = [e for e in eps if e["kind"] == "Target issue"]
|
|
for e in eps:
|
|
if e["kind"] == "Obstruction":
|
|
e["sensors"] = [s for s in e["sensors"] if not any(s in h["sensors"] and overlap(e, h) for h in issues)]
|
|
eps = [e for e in eps if e["kind"] != "Obstruction" or e["sensors"]]
|
|
# a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene
|
|
checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1]
|
|
eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)]
|
|
# the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several
|
|
# targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003)
|
|
hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal)
|
|
host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h))
|
|
and (h["label"] != "Heat shimmer" or h["start"] <= e["start"])
|
|
and (h["label"] != "Brief obstruction" or e["start"] == h["start"]
|
|
and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None)
|
|
for e in eps if e["label"] == JUMP_BACK}
|
|
# a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored,
|
|
# it takes its family (wf-improve session Displacement, mod. 1, S6)
|
|
host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e))
|
|
and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h)
|
|
and set(e["sensors"]) <= set(h["sensors"])), None))})
|
|
groups = {id(e): [e] for e in eps if not host.get(id(e))}
|
|
for e in eps:
|
|
if host.get(id(e)):
|
|
groups[id(host[id(e)])].append(e)
|
|
merged = []
|
|
for g in groups.values():
|
|
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue")
|
|
or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None)
|
|
end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None
|
|
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
|
|
prev += g
|
|
else:
|
|
merged.append(g)
|
|
return sorted((event(g, references) for g in merged), key=lambda v: v["start"])
|
|
|
|
|
|
def events(measurements, end=None, folder=None):
|
|
"""Measurements (data.load) -> events of all detectors: what the page receives. end: the end of the window read
|
|
(batch run), for a hole touching it; folder: the station folder, for the temperature and the camera pose. A detector
|
|
that fails is skipped with a message (a broken detector does not block the page)."""
|
|
episodes = []
|
|
for name in DETECTORS:
|
|
try:
|
|
kw = {"end": end} if name == "no_data" else _station(folder) if name == "thermal_displacement" else {}
|
|
episodes += importlib.import_module(name).detect(measurements, **kw)
|
|
except Exception as e: # noqa: BLE001
|
|
print(f"Detectors: {name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
|
|
return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
|
|
p.add_argument("folder")
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
|
|
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
|
|
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
|
|
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
|
|
# workflow as in the real application
|
|
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
|
|
a = p.parse_args()
|
|
measurements = load(a.folder)
|
|
ts = [m["t"] for ms in measurements.values() for m in ms]
|
|
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
|
|
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
|
|
t = start
|
|
while t < end:
|
|
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
|
|
h = t - timedelta(days=a.history)
|
|
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
|
|
evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else []
|
|
if a.chunk:
|
|
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
|
|
for v in evs:
|
|
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
|
|
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
|
|
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
|
|
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
|
|
t = b
|