2026-11-16 17:28:04 +01:00
|
|
|
"""From the detectors' episodes to events, one family each.
|
|
|
|
|
|
|
|
|
|
Each module of DETECTORS (next to this file) returns episodes: start, end, kind (the family), label (the variety), text,
|
|
|
|
|
ignore, sensors (a Poor visibility also has its level). classify() makes them events:
|
|
|
|
|
- a target already in an overlapping Target issue episode is dropped from an Obstruction episode: its curve is already
|
|
|
|
|
to be ignored;
|
|
|
|
|
- a jump back to normal (JUMP_BACK) overlapping a Visibility episode joins its event: rain, fog or heat shimmer
|
|
|
|
|
explains it; otherwise it stays Displacement, its cause to be checked on the images;
|
|
|
|
|
- a jump back whose targets are all in an overlapping Obstruction or Target issue episode joins it (something in front
|
|
|
|
|
of them explains it, or the curve of a problem target is to be ignored); a slow drift, in a Target issue episode;
|
|
|
|
|
- episodes of the same variety less than MERGE apart make one event (validated rule, 2026-10-24); an Obstruction or a
|
|
|
|
|
Target issue only with an episode of the same target (wf-improve Obstruction, mod. 1, S1; Target issue, mod. 1, S3).
|
|
|
|
|
The text of an event is that of its first episode, plus a sentence for the jumps it absorbed.
|
|
|
|
|
Usage: python3 scripts/categories.py <station folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk N] [--history N]
|
|
|
|
|
one line per event; --chunk: each chunk of N days classified alone, with the days of history before it; only the
|
|
|
|
|
events touching the chunk are shown, "nothing" for a quiet chunk.
|
2026-11-13 12:19:52 +01:00
|
|
|
"""
|
2026-11-16 16:06:24 +01:00
|
|
|
import importlib
|
2026-11-13 12:19:52 +01:00
|
|
|
import sys
|
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
2026-11-16 15:25:01 +01:00
|
|
|
# The detectors next to this file, and the single reader of the measurements: data.py of the tools repository, the
|
2026-11-16 17:34:59 +01:00
|
|
|
# sibling of this one (one copy of each repository in the home directory), else the tools/ link of the caller's root
|
|
|
|
|
# (a copy of this repository elsewhere, e.g. extracted by git archive).
|
2026-11-16 15:25:01 +01:00
|
|
|
HERE = Path(__file__).resolve().parent
|
2026-11-16 17:34:59 +01:00
|
|
|
sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts"), str(Path.cwd() / "tools" / "scripts")]
|
2026-11-16 17:43:12 +01:00
|
|
|
from episodes import targets # noqa: E402
|
2026-11-16 17:23:53 +01:00
|
|
|
from data import iso, load # noqa: E402
|
2026-11-13 12:19:52 +01:00
|
|
|
|
2026-11-16 17:28:04 +01:00
|
|
|
DETECTORS = ("displacement", "no_data", "target_issue", "thermal_displacement", "visibility")
|
|
|
|
|
# The families, in the order of the timeline rows; ignore: greyed zone (None: as its episodes say); process: Images
|
|
|
|
|
# button, nothing, or Images and a link to Diagnosis.
|
2026-11-13 12:19:52 +01:00
|
|
|
CATEGORIES = {
|
|
|
|
|
"Visibility": {"ignore": None, "process": "images"},
|
|
|
|
|
"Obstruction": {"ignore": True, "process": "images"},
|
|
|
|
|
"Target issue": {"ignore": True, "process": "images"},
|
|
|
|
|
"Thermal displacement": {"ignore": False, "process": "none"},
|
|
|
|
|
"No data": {"ignore": False, "process": "none"},
|
|
|
|
|
"Displacement": {"ignore": False, "process": "check"},
|
|
|
|
|
}
|
2026-11-16 17:32:00 +01:00
|
|
|
JUMP_BACK = "Sudden jump, back to normal" # the label of displacement.py
|
2026-11-16 17:28:04 +01:00
|
|
|
MERGE = timedelta(hours=1)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def variety(ep):
|
2026-11-16 17:28:04 +01:00
|
|
|
return ep["label"], ep.get("level")
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def overlap(a, b):
|
|
|
|
|
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 17:28:04 +01:00
|
|
|
def host(e, eps):
|
|
|
|
|
"""The episode whose event a jump back or a slow drift joins, or None."""
|
|
|
|
|
kinds = {JUMP_BACK: ("Obstruction", "Target issue"), "Slow drift": ("Target issue",)}.get(e["label"], ())
|
|
|
|
|
return next((h for h in eps if overlap(e, h) and (h["kind"] == "Visibility" and e["label"] == JUMP_BACK
|
|
|
|
|
or h["kind"] in kinds and set(e["sensors"]) <= set(h["sensors"]))), None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def event(group):
|
|
|
|
|
"""Episodes of an event, the main one first -> event."""
|
|
|
|
|
cat, c = group[0]["kind"], CATEGORIES[group[0]["kind"]]
|
|
|
|
|
main = [e for e in group if variety(e) == variety(group[0])] # the duration is theirs, not the absorbed jumps'
|
|
|
|
|
jumped = len({s for e in group if e["label"] == JUMP_BACK and e not in main for s in e["sensors"]})
|
|
|
|
|
text = group[0]["text"] + (f" The jumps of {targets(jumped)} in this period have the same cause." if jumped else "")
|
|
|
|
|
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat, "label": cat,
|
|
|
|
|
"text": text, "ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
|
|
|
|
|
"sensors": [] if any(not e["sensors"] for e in group) else sorted({s for e in group for s in e["sensors"]}),
|
|
|
|
|
"process": c["process"], "episodes": group}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def classify(episodes):
|
|
|
|
|
"""Episodes of all detectors -> events sorted by start (rules: module docstring)."""
|
2026-11-13 12:19:52 +01:00
|
|
|
eps = sorted(episodes, key=lambda e: e["start"])
|
2026-11-16 16:06:24 +01:00
|
|
|
issues = [e for e in eps if e["kind"] == "Target issue"]
|
|
|
|
|
for e in eps:
|
|
|
|
|
if e["kind"] == "Obstruction":
|
|
|
|
|
e["sensors"] = [s for s in e["sensors"] if not any(s in h["sensors"] and overlap(e, h) for h in issues)]
|
|
|
|
|
eps = [e for e in eps if e["kind"] != "Obstruction" or e["sensors"]]
|
2026-11-16 17:28:04 +01:00
|
|
|
hosts = {id(e): h for e in eps if (h := host(e, eps))}
|
|
|
|
|
groups = {id(e): [e] for e in eps if id(e) not in hosts}
|
2026-11-13 12:19:52 +01:00
|
|
|
for e in eps:
|
2026-11-16 17:28:04 +01:00
|
|
|
if id(e) in hosts:
|
|
|
|
|
groups[id(hosts[id(e)])].append(e)
|
2026-11-13 12:19:52 +01:00
|
|
|
merged = []
|
|
|
|
|
for g in groups.values():
|
2026-11-16 17:28:04 +01:00
|
|
|
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in (
|
|
|
|
|
"Obstruction", "Target issue") or set(g[0]["sensors"]) & set(m[0]["sensors"]))), None) # the same target
|
|
|
|
|
end = prev and max(e["end"] for e in prev if variety(e) == variety(g[0]))
|
2026-11-13 12:19:52 +01:00
|
|
|
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
|
|
|
|
|
prev += g
|
|
|
|
|
else:
|
|
|
|
|
merged.append(g)
|
2026-11-16 17:28:04 +01:00
|
|
|
return sorted((event(g) for g in merged), key=lambda v: v["start"])
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def events(measurements, end=None, folder=None):
|
2026-11-16 16:06:24 +01:00
|
|
|
"""Measurements (data.load) -> events of all detectors: what the page receives. end: the end of the window read
|
2026-11-16 17:28:04 +01:00
|
|
|
(batch run), for a hole touching it; folder: unused, kept for the callers. A detector that fails is skipped with a
|
|
|
|
|
message (a broken detector does not block the page)."""
|
2026-11-13 12:19:52 +01:00
|
|
|
episodes = []
|
2026-11-16 16:06:24 +01:00
|
|
|
for name in DETECTORS:
|
2026-11-13 12:19:52 +01:00
|
|
|
try:
|
2026-11-16 17:28:04 +01:00
|
|
|
episodes += importlib.import_module(name).detect(measurements, **({"end": end} if name == "no_data" else {}))
|
2026-11-13 12:19:52 +01:00
|
|
|
except Exception as e: # noqa: BLE001
|
2026-11-16 16:06:24 +01:00
|
|
|
print(f"Detectors: {name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
|
2026-11-16 17:28:04 +01:00
|
|
|
return classify(episodes)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
2026-11-17 09:32:20 +01:00
|
|
|
def window(measurements, start, end, history=timedelta(days=3)):
|
|
|
|
|
"""The events of [start, end) (UTC datetimes), the detectors given `history` of earlier measurements too (the rule of
|
|
|
|
|
the CLI below): those ending at or after start. For a caller (aimsight-app) as for the CLI."""
|
|
|
|
|
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if start - history <= m["t"] < end])}
|
|
|
|
|
return [v for v in events(chunk, end) if v["end"] >= iso(start)] if chunk else []
|
|
|
|
|
|
|
|
|
|
|
2026-11-13 12:19:52 +01:00
|
|
|
if __name__ == "__main__":
|
|
|
|
|
import argparse
|
|
|
|
|
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
|
|
|
|
|
p.add_argument("folder")
|
|
|
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
|
|
|
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
|
|
|
|
|
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
|
|
|
|
|
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
|
|
|
|
|
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
|
2026-11-13 12:26:02 +01:00
|
|
|
# workflow as in the real application
|
2026-11-13 12:19:52 +01:00
|
|
|
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
|
|
|
|
|
a = p.parse_args()
|
|
|
|
|
measurements = load(a.folder)
|
|
|
|
|
ts = [m["t"] for ms in measurements.values() for m in ms]
|
|
|
|
|
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
|
|
|
|
|
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
|
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
|
|
|
|
|
t = start
|
|
|
|
|
while t < end:
|
|
|
|
|
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
|
2026-11-17 09:32:20 +01:00
|
|
|
evs = window(measurements, t, b, timedelta(days=a.history))
|
2026-11-13 12:19:52 +01:00
|
|
|
if a.chunk:
|
|
|
|
|
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
|
|
|
|
|
for v in evs:
|
|
|
|
|
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
|
|
|
|
|
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
|
|
|
|
|
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
|
|
|
|
|
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
|
|
|
|
|
t = b
|