Files
aimsight-detectors/scripts/categories.py
T

141 lines
8.3 KiB
Python
Raw Normal View History

"""From the detectors' episodes to events, one family each.
Each module of DETECTORS (next to this file) returns episodes: start, end, kind (the family), label (the variety), text,
ignore, sensors (a Poor visibility also has its level). classify() makes them events:
- a target already in an overlapping Target issue episode is dropped from an Obstruction episode: its curve is already
to be ignored;
- a jump back to normal (JUMP_BACK) overlapping a Visibility episode joins its event: rain, fog or heat shimmer
explains it; otherwise it stays Displacement, its cause to be checked on the images;
- a jump back whose targets are all in an overlapping Obstruction or Target issue episode joins it (something in front
of them explains it, or the curve of a problem target is to be ignored); a slow drift, in a Target issue episode;
- episodes of the same variety less than MERGE apart make one event (validated rule, 2026-10-24); an Obstruction or a
Target issue only with an episode of the same target (wf-improve Obstruction, mod. 1, S1; Target issue, mod. 1, S3).
The text of an event is that of its first episode, plus a sentence for the jumps it absorbed.
Usage: python3 scripts/categories.py <station folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk N] [--history N]
one line per event; --chunk: each chunk of N days classified alone, with the days of history before it; only the
events touching the chunk are shown, "nothing" for a quiet chunk.
"""
import importlib
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
# The detectors next to this file, and the single reader of the measurements: data.py of the tools repository, the
# sibling of this one (one copy of each repository in the home directory), else the tools/ link of the caller's root
# (a copy of this repository elsewhere, e.g. extracted by git archive).
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts"), str(Path.cwd() / "tools" / "scripts")]
from episodes import targets # noqa: E402
from data import iso, load # noqa: E402
DETECTORS = ("displacement", "no_data", "target_issue", "thermal_displacement", "visibility")
# The families, in the order of the timeline rows; ignore: greyed zone (None: as its episodes say); process: Images
# button, nothing, or Images and a link to Diagnosis.
CATEGORIES = {
"Visibility": {"ignore": None, "process": "images"},
"Obstruction": {"ignore": True, "process": "images"},
"Target issue": {"ignore": True, "process": "images"},
"Thermal displacement": {"ignore": False, "process": "none"},
"No data": {"ignore": False, "process": "none"},
"Displacement": {"ignore": False, "process": "check"},
}
JUMP_BACK = "Sudden jump, back to normal" # the label of displacement.py
MERGE = timedelta(hours=1)
def variety(ep):
return ep["label"], ep.get("level")
def overlap(a, b):
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
def host(e, eps):
"""The episode whose event a jump back or a slow drift joins, or None."""
kinds = {JUMP_BACK: ("Obstruction", "Target issue"), "Slow drift": ("Target issue",)}.get(e["label"], ())
return next((h for h in eps if overlap(e, h) and (h["kind"] == "Visibility" and e["label"] == JUMP_BACK
or h["kind"] in kinds and set(e["sensors"]) <= set(h["sensors"]))), None)
def event(group):
"""Episodes of an event, the main one first -> event."""
cat, c = group[0]["kind"], CATEGORIES[group[0]["kind"]]
main = [e for e in group if variety(e) == variety(group[0])] # the duration is theirs, not the absorbed jumps'
jumped = len({s for e in group if e["label"] == JUMP_BACK and e not in main for s in e["sensors"]})
text = group[0]["text"] + (f" The jumps of {targets(jumped)} in this period have the same cause." if jumped else "")
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat, "label": cat,
"text": text, "ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
"sensors": [] if any(not e["sensors"] for e in group) else sorted({s for e in group for s in e["sensors"]}),
"process": c["process"], "episodes": group}
def classify(episodes):
"""Episodes of all detectors -> events sorted by start (rules: module docstring)."""
eps = sorted(episodes, key=lambda e: e["start"])
issues = [e for e in eps if e["kind"] == "Target issue"]
for e in eps:
if e["kind"] == "Obstruction":
e["sensors"] = [s for s in e["sensors"] if not any(s in h["sensors"] and overlap(e, h) for h in issues)]
eps = [e for e in eps if e["kind"] != "Obstruction" or e["sensors"]]
hosts = {id(e): h for e in eps if (h := host(e, eps))}
groups = {id(e): [e] for e in eps if id(e) not in hosts}
for e in eps:
if id(e) in hosts:
groups[id(hosts[id(e)])].append(e)
merged = []
for g in groups.values():
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in (
"Obstruction", "Target issue") or set(g[0]["sensors"]) & set(m[0]["sensors"]))), None) # the same target
end = prev and max(e["end"] for e in prev if variety(e) == variety(g[0]))
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
prev += g
else:
merged.append(g)
return sorted((event(g) for g in merged), key=lambda v: v["start"])
def events(measurements, end=None, folder=None):
"""Measurements (data.load) -> events of all detectors: what the page receives. end: the end of the window read
(batch run), for a hole touching it; folder: unused, kept for the callers. A detector that fails is skipped with a
message (a broken detector does not block the page)."""
episodes = []
for name in DETECTORS:
try:
episodes += importlib.import_module(name).detect(measurements, **({"end": end} if name == "no_data" else {}))
except Exception as e: # noqa: BLE001
print(f"Detectors: {name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
return classify(episodes)
if __name__ == "__main__":
import argparse
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
p.add_argument("folder")
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
# workflow as in the real application
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
a = p.parse_args()
measurements = load(a.folder)
ts = [m["t"] for ms in measurements.values() for m in ms]
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
t = start
while t < end:
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
h = t - timedelta(days=a.history)
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else []
if a.chunk:
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
for v in evs:
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
t = b