Detectors in English, under scripts/; learner tools under scripts/learner/

- data.py (was donnees.py): unused filtrer, heure_image and image sorting removed
- score.py: dead functions (validite, marquer, detectes, score, familles) and the
  validity levels removed; --from/--to
- regression.py: history.csv and --seance/--etape removed
- categories.py: --from/--to/--chunk/--history; process values images/none/check
- benchmark on all stations unchanged (same 132 figures)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-13 12:19:52 +01:00
1 parent 9b3ac0ecfe
commit 969637499f
23 files changed
+1877 -1971

No files matched your search

+243
View File
@@ -0,0 +1,243 @@
"""From the detectors' episodes to events, one per family.
One detector per label family (labeling/familles.md): visibility.py, obstruction.py, target_issue.py,
thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its
episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules:
- a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat
shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and
family; otherwise it stays Displacement, cause not established, to be checked on the images (familles.md, "A jump then
the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several
days it crosses (learner session Displacement, mod. 1, S5);
- a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the
curve of a problem target is to be ignored, familles.md);
- a shift that lasts, or too recent to know (SHIFTS), is never absorbed;
- heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of
a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back
starting during it, not the one of the day before coming back within its day (learner session Visibility, mod. 1,
S1: WILDSTRUBEL);
- two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction)
only if they share a sensor (learner session Obstruction, mod. 1: S1, different targets chained into an event of
several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its
bounds (learner session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after
its instant (S3: not the 30 h jump of another target).
Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client"
column of familles.md says; the cause is only given if the variety establishes it, otherwise the images show it.
No detect function: the loader does not take this file for a detector.
Usage: python3 detectors/scripts/categories.py <station folder> (one line per event: family, variety, sensors)
... <folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days
classified alone, with the days of history before it if they exist; only the events touching the chunk are
shown; "nothing" for a quiet chunk)
"""
import functools
import importlib.util
import inspect
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402
# What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134):
# temperature [(t, °C)], hourly median of the components; camera [pose] (pan, tilt, rotation, scale, sigma0);
# metrics {name: [(t, value)]}, all numeric metrics (sensor_CPU_deg_c, sensor_GPU_deg_c, BusVoltage…).
CONTEXT = {"temperature": load_temperature, "camera": load_camera, "metrics": load_metrics}
@functools.lru_cache(maxsize=None)
def _context(folder, key):
return CONTEXT[key](folder)
# The families of labeling/familles.md, in the order of the timeline rows; ignore: greyed zone (None: as its episodes
# say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to
# Diagnosis.
CATEGORIES = {
"Visibility": {"ignore": None, "process": "images"},
"Obstruction": {"ignore": True, "process": "images"},
"Target issue": {"ignore": True, "process": "images"},
"Thermal displacement": {"ignore": False, "process": "none"},
"No data": {"ignore": False, "process": "none"},
"Displacement": {"ignore": False, "process": "check"},
}
JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement
SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed
HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction",
"Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an
# overlapping jump back
MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24)
def detectors():
"""Modules of the *.py next to this file that have detect; a file that does not load is skipped with a message."""
mods = []
for path in sorted(Path(__file__).resolve().parent.glob("*.py")):
if path == Path(__file__).resolve():
continue
try:
spec = importlib.util.spec_from_file_location(f"detector_{path.stem}", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
except Exception as e: # noqa: BLE001 (a broken detector does not block the page)
print(f"Detectors: {path.name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
continue
if hasattr(mod, "detect"):
mods.append(mod)
return mods
def variety(ep):
"""The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of
visibility by level."""
label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize()
return ep["kind"], label, ep.get("level")
def is_thermal(ep):
return ep["kind"] == "Thermal displacement"
def overlap(a, b):
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
def targets(n):
return "one target" if n == 1 else f"{n} targets"
def sentence(group, sensors, references):
"""Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed
jump adds that it has the same cause."""
n, one, ep = len(sensors), len(sensors) == 1, group[0]
jumps = [e for e in group[1:] if e["label"] == JUMP_BACK]
jumped = len({s for e in jumps for s in e["sensors"]})
lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])})
refs = ", including the reference targets" if references & set(sensors) else ""
camera = ep["label"].startswith("Camera")
large = any(e["label"] == "Large shift" for e in group)
sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"]
text = {
"Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. "
"The curves in this period are not reliable: ignore them.",
"Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. "
"The data in this period are less precise: use them with care.",
"Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.",
"Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around "
f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.",
"Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.",
"expansion": ("The camera support expands in the day heat; the curves are corrected for it."
if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.",
"contraction": ("The camera support contracts as it cools down; the curves are corrected for it."
if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.",
JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.",
"shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large
else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting
else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n
else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.")
+ " Check whether the structure moved: look at the images.",
}.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text
if ep["kind"] == "Target issue":
text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target."
if not jumped:
return text
return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1
else f" The jumps of {targets(jumped)} in this period have the same cause.")
def event(group, references):
"""Episodes of the same event, the main one first -> event."""
cat, var = group[0]["kind"], variety(group[0])
c = CATEGORIES.get(cat, {"ignore": None, "process": "none"})
sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"]))
main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps'
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat,
"label": cat, "text": sentence(group, sensors, references),
"ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
"sensors": sensors, "process": c["process"], "episodes": group}
def classify(episodes, references=frozenset()):
"""Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref
sensors, for the sentence."""
eps = sorted(episodes, key=lambda e: e["start"])
# a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene
checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1]
eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)]
# the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several
# targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003)
hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal)
host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h))
and (h["label"] != "Heat shimmer" or h["start"] <= e["start"])
and (h["label"] != "Brief obstruction" or e["start"] == h["start"]
and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None)
for e in eps if e["label"] == JUMP_BACK}
# a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored,
# it takes its family (learner session Displacement, mod. 1, S6)
host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e))
and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h)
and set(e["sensors"]) <= set(h["sensors"])), None))})
groups = {id(e): [e] for e in eps if not host.get(id(e))}
for e in eps:
if host.get(id(e)):
groups[id(host[id(e)])].append(e)
merged = []
for g in groups.values():
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue")
or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None)
end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
prev += g
else:
merged.append(g)
return sorted((event(g, references) for g in merged), key=lambda v: v["start"])
def events(measurements, end=None, folder=None):
"""Measurements (data.load) -> events of all detectors: what the page receives. A detector also receives what its
detect function declares: end, the end of the window read (batch run), for a hole touching it; a CONTEXT key, read
from the station folder (once per folder), empty without a folder."""
episodes = []
for mod in detectors():
try:
params = inspect.signature(mod.detect).parameters
kw = {k: _context(folder, k) if folder else {} if k == "metrics" else [] for k in CONTEXT if k in params}
if "end" in params:
kw["end"] = end
episodes += mod.detect(measurements, **kw)
except Exception as e: # noqa: BLE001
print(f"Detectors: {mod.__name__} skipped ({type(e).__name__}: {e})", file=sys.stderr)
return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"})
if __name__ == "__main__":
import argparse
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
p.add_argument("folder")
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
# learner as in the real application
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
a = p.parse_args()
measurements = load(a.folder)
ts = [m["t"] for ms in measurements.values() for m in ms]
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
t = start
while t < end:
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
h = t - timedelta(days=a.history)
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else []
if a.chunk:
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
for v in evs:
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
t = b
+140
View File
@@ -0,0 +1,140 @@
"""Reading a station folder (standard library only). Every time is UTC.
Column formats: labeling/connaissances/colonnes-des-deplacements.md in Diagnostic-app.
"""
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
def _header(path):
with open(path, newline="") as f:
return next(csv.reader(f), [])
def find_files(folder):
"""Station folder -> {'advanced': [...], 'camera': [...]}: sorted CSV paths, subfolders included."""
out = {"advanced": [], "camera": []}
for p in sorted(glob.glob(os.path.join(folder, "**", "*.csv"), recursive=True)):
header = _header(p)
if "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
out["advanced"].append(p)
elif "median_pan" in header and "timestamp" not in header: # timestamp: the hourly export
out["camera"].append(p)
return out
def _number(v):
v = v.strip()
return float(v) if v else None
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
"""
by = {}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
m["t"] = parse_time(r["time_utc"])
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together; the median drops the spike of a single
one, the computing GPU (D-124)."""
by = {}
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
if (v := _number(r.get("metric") or "")) is not None:
by.setdefault(parse_time(r["time_utc"]), []).append(v)
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
def load_camera(folder):
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
A measurement instant without a pose: the camera was not registered.
"""
by = {}
for path in find_files(folder)["camera"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
p["t"] = parse_time(r["time_utc"])
by[p["t"]] = p
return [by[t] for t in sorted(by)]
def load_metrics(folder):
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
read. Name: the file's, without the station or the date."""
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
if not (m := METRIC_RE.match(os.path.basename(path))):
continue
name = m.group(1).removeprefix(station)
if IDENTIFIERS.match(name):
continue
with open(path, newline="") as f:
for r in csv.DictReader(f):
try:
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
except ValueError:
continue
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
return {n: sorted(d.items()) for n, d in sorted(by.items())}
def is_reference(measurements):
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
none, filled reproj_err or inlier (references only)."""
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
def rejected(m):
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
two values, an empty corr_x."""
if m["error_value"] is not None and m["error_threshold"] is not None:
return m["error_value"] > m["error_threshold"]
return m["corr_x"] is None
def parse_time(text):
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
t = datetime.fromisoformat(text)
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
def iso(t):
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
+327
View File
@@ -0,0 +1,327 @@
"""Detector of the Displacement family (labeling/familles.md): sudden jumps of corr_x / corr_y (px) of each target, at
the native cadence of the "advanced" export, and slow drift (end of file). A flipping target ("Target flipping")
belongs to the Target issue family: episodes() returns it, detect() leaves it to target_issue.py.
1. Noise of the series = median of the gaps from one measurement to the next; jump = measurement K noises or more away
from the level before (median of the BASE previous measurements); the first BASE measurements, lacking a level
before, are compared with the level after (median of the BASE next ones), a gap there is a jump back at once.
2. Back (an hour of measurements in a row within half the jump: BASE at 15 min, a single one at 4 h; learner
MICHIGAN, mod. 1) within a day, or one day later back at its usual place (first day of the export) or at the day
before's: "Sudden jump, back to normal", greyed; its cause is not established here (categories.py gives it the
cause of the category that overlaps it, otherwise: to be checked on the images).
3. Otherwise "Lasting shift", never greyed (maybe a real movement); less than a day of hindsight: "Jump, not back yet",
not greyed either.
4. Jumps following each other within GAP, all targets together, make a single episode (common mode); a single lasting
(or too recent) target is enough not to grey the episode.
5. Global references excluded; rejected measurements (empty corr) skipped.
6. Common peak (NORDEND, batch 01): at the same instant, at least PEAK_MIN targets move away by at least K_PEAK noises
from both neighbouring measurements, on the same side (the whole image shaken for one step): a jump back for each,
even under K. It joins the episodes of point 4 (next to a lasting jump, it greys nothing).
7. (CLARIDEN, batch 01) a jump that never comes back no longer ends the reading of the series, its new level becomes
the level before; an episode that is not greyed ends at its last jump, not at the returns; a flipping target
(REPEAT jumps back or more, each within a day of the previous one) makes a "back to normal" episode on its own, from
the first to the last flip, outside the common episodes. A target that flips for at least FLIP, with at most
MAX_FLIPPING targets at the same time, becomes "Target flipping" (Target issue, D-074); a lasting jump of at least
LARGE of the targets, "Large shift" (everything moved), always to be checked. Learner session Target issue, mod. 1,
S2: FLIP from 6 h to 36 h; a target flipping for a day during a nuisance of the whole scene (rain, fog, night of poor
visibility) is not a problem target; its jumps back go to the nuisance overlapping them, otherwise stay
Displacement.
8. Jump under K that stays (NIEDERHORN, batch 01): a jump of at least K_LASTING noises counts if it is lasting in the
sense of point 3 and the median of the day after moves away from that of the day before, on the jump's side, by at
least PLATEAU times the ordinary variation of the level from one UTC day to the next (median of the gaps between
daily levels); otherwise it is ignored (neither back nor too recent: only jumps of K noises or more are).
"""
import statistics
import sys
from datetime import timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent)) # data.py, next to it
import data # noqa: E402
K = 20 # tuned on E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 (17 and 19
# noises) are caught by visibility.py: do not lower K
K_LASTING = 7 # under K, a jump that stays counts from K_LASTING noises (NIEDERHORN, batch 01: e09, corr_y −0.18 px at
# once, 8 noises, stayed 5 days)
PLATEAU = 8 # ... if it stays: median of the day after, on the jump's side, PLATEAU times the ordinary variation of the
# level from one day to the next away from the day before (e09: 13; E-001, healthy targets: 6 at most)
K_PEAK = 6 # one-step peak common to several targets (NORDEND shakes, 6 to 11 noises, batch 01)
PEAK_MIN = 3 # targets peaking at the same step for a common peak (one or two: ordinary noise)
BASE = 4 # measurements that make the level before (1 h at 15 min)
GAP = timedelta(hours=1) # two jumps within an hour, all targets together: a single episode
BACK_H = 24 # back within a day = artefact (E-002: plateaus up to 11.5 h)
FLIP = timedelta(hours=36) # a problem target flips for several days (PR_BG: 2.5 days); less is an agitated morning
# (E-009: C8 and C13, 5 flips in 1 h 20 during the movement) or a night of nuisance (NORDEND, JUNGFRAU:
# 12 to 28 h within a loss of visibility), a jump back, absorbed by its host
MAX_FLIPPING = 2 # targets flipping at the same time, at most (FLORIDA: the two stuck ones; E-002, cobweb on the
# references: 15 at once)
LARGE = 0.5 # share of the targets of a lasting jump for a large shift (E-004: almost all)
REPEAT = 5 # jumps back of the same target, each within a day of the other, that make it a flipping target (PR_BG of
# CLARIDEN, batch 01: about thirty in 2.5 days)
def _jumps(t, v):
"""A series (instants, values) -> [(start i, end i, state)], state 'back', 'lasting' or 'recent'."""
day = timedelta(hours=BACK_H)
noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:]))
step = statistics.median(b - a for a, b in zip(t, t[1:])) # cadence of the series
back_n = max(1, min(BASE, round(timedelta(hours=1) / step))) # measurements of an hour (BASE at 15 min, one at 4 h)
# ponytail: usual place = first day of the export, wrong if it starts within an artefact
ref = statistics.median(x for x, u in zip(v, t) if u < t[0] + day)
lv = {} # level of each UTC day; its ordinary variation from one day to the next judges a jump under K
for x, u in zip(v, t):
lv.setdefault(u.date(), []).append(x)
lv = [statistics.median(x) for _, x in sorted(lv.items())]
ordinary = statistics.median(abs(b - a) for a, b in zip(lv, lv[1:])) if len(lv) > 2 else 0
# the first BASE measurements, lacking a level before, face the level after: a gap there came back at once
out = [(k, k, "back") for k in range(min(BASE, len(v) - BASE))
if noise and abs(v[k] - statistics.median(v[k + 1:k + 1 + BASE])) >= K * noise]
i = BASE
while noise and i < len(v):
base = statistics.median(v[i - BASE:i])
if abs(v[i] - base) < K_LASTING * noise:
i += 1
continue
tol = abs(v[i] - base) / 2 # back: an hour of measurements in a row within half the jump from the level before
j = next((k for k in range(i + 1, len(v)) if all(abs(x - base) < tol for x in v[k:k + back_n])), len(v))
before = statistics.median([x for x, u in zip(v, t) if t[i] - day <= u < t[i]] or [base])
after = statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + day)
if j < len(v) and t[j] - t[i] <= day:
state = "back"
elif abs(after - ref) < tol or abs(after - before) < tol: # the day after, the target is at its usual place,
state, j = "back", i + 1 # or where it was the day before: the jump ended a dip
else:
state = "lasting" if t[-1] - t[i] >= day else "recent" # less than a day of hindsight: not known yet
if abs(v[i] - base) < K * noise and (state != "lasting" or (after - before) * (v[i] - base) < 0
or not ordinary or abs(after - before) < PLATEAU * ordinary):
i += 1 # under K, only a jump that stays counts: the day after far from the day before, on its side
continue
out.append((i, j - 1, state))
# level before rebuilt after the return; never back: the new level becomes the level before, the rest is
# still read (PR_BG of CLARIDEN, batch 01: its flips after the 10:30 jump were invisible)
i = (j if j < len(v) else i) + BASE
return out
def _peaks(t, v):
"""Instants where the measurement moves away by at least K_PEAK noises from both neighbours, same side (one-step peak)."""
noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:]))
return [t[i] for i in range(1, len(v) - 1) if noise and (v[i] - v[i - 1]) * (v[i] - v[i + 1]) > 0
and min(abs(v[i] - v[i - 1]), abs(v[i] - v[i + 1])) >= K_PEAK * noise]
def episodes(measurements):
"""{sensor: [measurement]} (data.load) -> jump episodes, sorted by start: Displacement, and Target issue for a
flipping target."""
found = [] # (start, end, state, sensor)
peaks = {} # instant -> targets peaking there for one step
for name, ms in measurements.items():
if data.is_reference(ms):
continue
for axis in ("corr_x", "corr_y"):
kept = [m for m in ms if m[axis] is not None]
if len(kept) <= BASE:
continue
t, v = [m["t"] for m in kept], [m[axis] for m in kept]
# a jump that did not come back ends with itself, not with the data (its bar hid the others)
found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)]
for u in _peaks(t, v):
peaks.setdefault(u, set()).add(name)
found += [(u, u, "back", s) for u, cs in peaks.items() if len(cs) >= PEAK_MIN for s in cs] # common peak: back
alone = _flips(found) # a target that keeps jumping and coming back: an episode on its own, outside the common ones
found = [f for f in found if f[2] != "back" or not any(f[3] in g["sensors"] and g["start"] <= f[0] <= g["end"]
for g in alone)]
groups = [] # a jump that did not come back only counts by its instant: its end is often the end of the data
for s, e, state, name in sorted(found):
if groups and s <= groups[-1]["until"] + GAP:
g = groups[-1]
else:
g = {"start": s, "end": e, "until": s, "last": s, "sensors": {}}
groups.append(g)
g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s
g["sensors"].setdefault(name, set()).add(state)
long = [g for g in alone if g["flip"]] # flips of many targets at the same time: the compensation, not a
for g in long: # problem target
if sum(h["start"] <= g["end"] and h["end"] >= g["start"] for h in long) > MAX_FLIPPING:
g["flip"] = 0
total = sum(not data.is_reference(ms) for ms in measurements.values())
return sorted((_episode(g, total) for g in groups + alone), key=lambda ep: ep["start"])
def _flips(found):
"""Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> a group, from the first
to the last flip."""
day, alone = timedelta(hours=BACK_H), []
for name in {f[3] for f in found}:
chain = [] # (start, end); corr_x and corr_y twice: counted by instant
for s, e in sorted({(s, e) for s, e, state, n in found if n == name and state == "back"}) + [(None, None)]:
if chain and s is not None and s <= max(c[1] for c in chain) + day:
chain.append((s, e))
continue
if len({c[0] for c in chain}) >= REPEAT:
alone.append({"start": chain[0][0], "end": max(c[1] for c in chain), "sensors": {name: {"back"}},
"flip": len({c[0] for c in chain}) if max(c[1] for c in chain) - chain[0][0] >= FLIP else 0})
chain = [(s, e)]
return alone
def _episode(g, total):
"""A group -> episode; a single lasting (or too recent) jump is enough not to grey; a flipping target: "Target
flipping" (Target issue, D-074); a lasting jump of LARGE targets or more: "Large shift"."""
n = len(g["sensors"])
lasting = sum("lasting" in st for st in g["sensors"].values())
recent = any("recent" in st for st in g["sensors"].values())
who = "One target" if n == 1 else f"{n} targets together (a single event)"
if g.get("flip"):
name = next(iter(g["sensors"]))
return {"start": data.iso(g["start"]), "end": data.iso(g["end"]), "kind": "Target issue", "label": "Target flipping",
"text": f"{name} jumped and came back {g['flip']} times: its tracking is not reliable, look at the images "
"of the target.", "ignore": True, "sensors": [name], "lasting": []}
if lasting and n >= LARGE * total:
label = "Large shift"
text = f"{n} targets jumped together and {lasting} of them stayed shifted: everything moved, please check it."
elif lasting:
label = "Lasting shift"
some = "" if lasting == n else f", {lasting} of them stayed shifted for more than a day"
text = (f"{who} jumped{some}" if some else f"{who} jumped and stayed shifted for more than a day") + \
": this may be a real movement, please check it."
elif recent:
label = "Jump, not back yet"
text = f"{who} jumped less than a day before the last measurement and has not come back yet: check again later."
else:
label = "Sudden jump, back to normal"
text = f"{who} jumped and came back within a day: the cause is not established, look at the images."
end = g["last"] if lasting or recent else g["end"] # not greyed: the last jump, not the returns
return {"start": data.iso(g["start"]), "end": data.iso(end), "kind": "Displacement", "label": label,
"text": text, "ignore": not lasting and not recent, "sensors": sorted(g["sensors"]),
"lasting": sorted(s for s, st in g["sensors"].items() if "lasting" in st)} # stayed shifted (event sentence)
def detect(measurements):
return sorted([e for e in episodes(measurements) if e["kind"] == "Displacement"] + drift(measurements),
key=lambda ep: ep["start"])
# --- Slow drift ------------------------------------------------------------------------------------------------------
# A target that moves little by little, day after day. Takes up the trend of the Displacements tab (D-055) on the
# "advanced" export (corr_x, corr_y, px). Method, per target (global references excluded) and per axis:
# 1. Level of the day: median of the measurements of each UTC day with at least HOURS measured hours out of 24, as a
# share of the hours the station measures (learner MICHIGAN, mod. 1: at one measurement every 4 h, 5 of the 6 hours;
# before, no day counted); at least DAYS days.
# 2. Theil-Sen slope and Kendall tau of the daily levels; amplitude of the day: gap between its 9th and 1st deciles.
# 3. Drift: |slope| × duration ≥ SIZE times the median daily amplitude, |tau| ≥ STEADY, and no step from one day to the
# next of more than half the drift (a jump that lasts is not a drift: the jump rule takes it).
# 4. One episode per target (one axis is enough), from the first measurement of the first day to the last of the last
# day; not greyed.
# 4b. Piecewise (learner session Displacement, mod. 1, S1): a target without a drift over the whole window may drift for
# a few days, then settle or go back. On each run of at least DAYS kept days, drift = level of the last day minus
# that of the first, at least PIECE daily amplitudes (median of the series), |tau| ≥ STEADY with the same sign, no
# step from one kept day to the next of more than half the drift; pieces taken from the largest drift to the
# smallest, without overlapping (they may share the day where the level turns): each goes from one extreme of the
# level to the other, one episode per piece and per axis. At PIECE = 1, the thermal cycles of a few days and the
# noise of the levels get through (JUNGFRAU, NORDEND): 3.5 keeps the rows of CLARIDEN (3.8 to 8.3 amplitudes),
# drops JUNGFRAU (3.1 at most). The pieces do not replace step 3. A piece does not cross a hole: two consecutive kept
# days are at most MAX_HOLE days apart (mod. 2: the gap of a level across three days without a measurement is a
# step, not a slope; it made pieces of the whole window, CLARIDEN 09-28 → 11-01). A missing day does not cut.
# 5. Camera drift (same mod.): at least LARGE of the targets drift on the same side on corr_x or corr_y, and at least
# LARGE of the sensors, references included, on their raw positions x or y: only the pieces of point 4b are
# reported, not the trend over the whole window (mod. 1, S2: the badly compensated drifting camera, NEBRASKA, is a
# Displacement for the labels). The targets of a bridge following the season (JUNGFRAU, labelled Thermal
# displacement day after day) pass step 3 but not PIECE; NEBRASKA both.
HOURS = 20 # a day counts if it has at least 20 measured hours out of 24 (as D-055), as a share of the measured hours
DAYS = 5 # minimum days for a trend (D-055)
SIZE = 1.0 # drift ≥ SIZE daily amplitudes (D-055)
STEADY = 0.6 # |Kendall tau| of the daily levels (D-055)
PIECE = 3.5 # piecewise drift ≥ PIECE daily amplitudes (point 4b)
MAX_HOLE = 2 # a piece does not go from one kept day to the next more than MAX_HOLE days apart (point 4b)
def _decile(vs, q):
vs = sorted(vs)
return vs[min(len(vs) - 1, int(q * len(vs)))]
def _days(ms, col):
"""-> {UTC day: its measurements}, the kept days (point 1)."""
days = {}
for m in ms:
if m[col] is not None:
days.setdefault(m["t"].date(), []).append(m)
slots = len({m["t"].hour for dm in days.values() for m in dm}) # hours the station measures (6 at a 4 h step)
return {d: dm for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots}
def _amplitude(days, col):
return statistics.median(_decile([m[col] for m in dm], 0.9) - _decile([m[col] for m in dm], 0.1) for dm in days.values())
def _drift(ms, col):
"""-> (drift in px, days, first instant, last) or None."""
days = _days(ms, col)
if len(days) < DAYS:
return None
lv = sorted((d.toordinal(), statistics.median(m[col] for m in dm)) for d, dm in days.items())
pairs = [(a, b) for k, a in enumerate(lv) for b in lv[k + 1:]]
slope = statistics.median((b[1] - a[1]) / (b[0] - a[0]) for a, b in pairs)
tau = sum((b[1] > a[1]) - (b[1] < a[1]) for a, b in pairs) / len(pairs)
drift = slope * (lv[-1][0] - lv[0][0])
step = max(abs(b[1] - a[1]) for a, b in zip(lv, lv[1:]))
if abs(drift) < SIZE * _amplitude(days, col) or abs(tau) < STEADY or step > abs(drift) / 2:
return None
d0, d1 = min(days), max(days)
return drift, len(days), min(m["t"] for m in days[d0]), max(m["t"] for m in days[d1])
def _pieces(ms, col):
"""-> [(drift in px, days, first instant, last)]: the drift pieces (point 4b)."""
days = _days(ms, col)
if len(days) < DAYS:
return []
ds = sorted(days)
lv = [statistics.median(m[col] for m in days[d]) for d in ds]
ampl = _amplitude(days, col)
pieces = [] # (drift, i, j)
for i in range(len(ds)):
signs = step = 0
for j in range(i + 1, len(ds)):
if (ds[j] - ds[j - 1]).days > MAX_HOLE:
break # a gap across a hole is a step (the jump rule takes it), not a slope
signs += sum((lv[j] > lv[k]) - (lv[j] < lv[k]) for k in range(i, j)) # Kendall tau, step by step
step = max(step, abs(lv[j] - lv[j - 1]))
n, d = j - i + 1, lv[j] - lv[i]
if n >= DAYS and d and abs(d) >= PIECE * ampl and step <= abs(d) / 2 \
and signs * (1 if d > 0 else -1) >= STEADY * n * (n - 1) / 2:
pieces.append((d, i, j))
taken = []
for d, i, j in sorted(pieces, key=lambda x: (-abs(x[0]), x[2] - x[1])):
if all(j <= a or i >= b for _, a, b in taken):
taken.append((d, i, j))
return [(d, j - i + 1, min(m["t"] for m in days[ds[i]]), max(m["t"] for m in days[ds[j]])) for d, i, j in taken]
def _common(measurements, cols):
"""At least LARGE of the sensors drift on the same side on the same axis of cols."""
for col in cols:
signs = [x[0] > 0 for ms in measurements.values() if (x := _drift(ms, col))]
if max(signs.count(True), signs.count(False)) >= LARGE * len(measurements):
return True
return False
def drift(measurements):
targets = {s: ms for s, ms in measurements.items() if not data.is_reference(ms)}
camera = _common(targets, ("corr_x", "corr_y")) and _common(measurements, ("x", "y")) # point 5: pieces only
out = []
for s, ms in sorted(targets.items()):
r = [] if camera else [x for col in ("corr_x", "corr_y") if (x := _drift(ms, col))]
r = [max(r, key=lambda x: abs(x[0]))] if r else [x for col in ("corr_x", "corr_y") for x in _pieces(ms, col)]
out += [{"start": data.iso(a), "end": data.iso(b), "kind": "Displacement", "label": "Slow drift",
"text": f"{s} drifts slowly and steadily: {d:+.2f} px in {n} days. Keep watching it.",
"ignore": False, "sensors": [s]} for d, n, a, b in r]
return out
if __name__ == "__main__":
for ep in detect(data.load(sys.argv[1])):
print(ep["start"], ep["end"], "ignore" if ep["ignore"] else "KEEP ", ep["label"], "|", len(ep["sensors"]),
",".join(ep["sensors"][:6]), "|", ep["text"])
+49
View File
@@ -0,0 +1,49 @@
"""Benchmark of the learner (D-116): the detectors against all the labeller's labels (labeling/auto/).
For each station of labeling/auto/ that has its folder in data/, the score per family and per instant of score.py
(D-112), over the range of its labels: from the day of the first to the midnight after the last. Then the total, family
by family, which says whether the detectors do better overall from one session to the next. The block of the session's
station is its score; the other blocks, the non-regression: a family that goes back on a station is a regression, even
if the total rises.
Usage: python3 detectors/scripts/learner/regression.py [--stations A,B]
"""
import argparse
import json
from datetime import timedelta
import score # next to it
from data import iso, parse_time # on the path through score
def label_range(path):
"""labeling/auto/<STATION>.jsonl -> (start, end) iso, in whole UTC days, end excluded."""
with open(path, encoding="utf-8") as f:
labels = [json.loads(l) for l in f if l.strip()]
start = min(e["debut"] for e in labels)[:10]
# ponytail: the end of the batch is not written; midnight closest to the last bound (a label bound overflows by one
# measurement after midnight); the real end of the batch in labeling/auto/ if the gap matters
end = iso(parse_time(max(e["fin"] for e in labels)) + timedelta(hours=12))[:10]
return iso(parse_time(start)), iso(parse_time(end))
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("--stations", help="A,B: these stations only (a shorter run)")
a = p.parse_args()
total = {}
for path in sorted((score.WORK / "labeling" / "auto").glob("*.jsonl")):
station = path.stem
if a.stations and station not in a.stations.split(","):
continue
folder = score.WORK / "data" / station
if not folder.is_dir():
print(f"{station}: no data/{station}/, skipped\n")
continue
start, end = label_range(path)
n, labels = score.family_scores(str(folder), start, end)
score.show(f"{station}: {start} → {end}, {len(labels)} labels", n)
print()
for k, v in n.items():
total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)]
score.show("Total", total)
+97
View File
@@ -0,0 +1,97 @@
"""Score of the detectors on a range: the detectors' events against the labels of labeling/auto/<STATION>.jsonl that
touch the range, per family (labeling/familles.md) and per instant (D-112). An instant: an hour of the range where at
least one sensor has a measurement; it receives, on one side, the families of the labels covering it, bounds excluded
(except for a hole, without a measurement inside), on the other, those of the detectors' events (categories.py, the
range classified with its 3 days of history). The category of an event is its family.
found: labelled instants that the detectors also flagged (share of the labelled ones);
correct: flagged instants that a label also puts in the family (share of the flagged ones).
The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error.
Label fields (written by Diagnostic-app): debut, fin, categorie (the family), description.
Run from the working repository, where detectors/ is mounted.
Usage: python3 detectors/scripts/learner/score.py <station folder> --from YYYY-MM-DD --to YYYY-MM-DD
"""
import argparse
import json
import os
import sys
from datetime import timedelta
from pathlib import Path
REPO = Path(os.path.abspath(__file__)).parents[2] # the detectors repository
WORK = REPO.parent # the working repository (labeling/, data/)
sys.path.insert(0, str(REPO / "scripts"))
import categories # noqa: E402
from data import iso, load, parse_time # noqa: E402
HISTORY = timedelta(days=3) # as the batch command (D-056)
def events(measurements, start, end, folder=None):
"""Detectors' events touching the range, classified with its days of history (as the batch command)."""
h, b = parse_time(start) - HISTORY, parse_time(end)
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start]
def count(pairs):
"""[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}."""
n = {}
for a, d in pairs:
for k in a | d:
x = n.setdefault(k, [0, 0, 0])
x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d)
return n
def auto_labels(station, start, end):
"""labeling/auto/<STATION>.jsonl -> the labeller's labels touching [start, end)."""
path = WORK / "labeling" / "auto" / f"{station}.jsonl"
if not path.exists():
return []
with open(path, encoding="utf-8") as f:
return [e for e in map(json.loads, filter(str.strip, f)) if e["debut"] < end and e["fin"] >= start]
def family_scores(folder, start, end):
"""Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels)."""
# ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the
# label if this precision is missing
measurements = load(folder)
labels = auto_labels(os.path.basename(os.path.normpath(folder)), start, end)
evts = events(measurements, start, end, folder)
instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end})
# bounds excluded: they are the healthy measurement before and after (D-111); a hole, without a measurement
# inside, keeps them
empty = {i for i, e in enumerate(labels) if not any(e["debut"] < t < e["fin"] for t in instants)}
pairs = []
for t in instants:
a = {e.get("categorie", "no family") for i, e in enumerate(labels)
if e["debut"] < t < e["fin"] or (i in empty and e["debut"] <= t <= e["fin"])} - {"None"}
d = {v["category"] for v in evts if v["start"] <= t <= v["end"]}
pairs.append((a, d))
return count(pairs), labels
def pct(a, b):
return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—"
def show(title, n):
print(title)
print(f" {'':<22}{'found':>20}{'correct':>20}")
for k in sorted(n):
labelled, flagged, both = n[k]
print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}")
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("folder")
p.add_argument("--from", dest="start", required=True)
p.add_argument("--to", dest="end", required=True)
a = p.parse_args()
station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end))
n, labels = family_scores(a.folder, start, end)
if not labels:
sys.exit(f"No label of {station} on {start} → {end} in labeling/auto/{station}.jsonl: analyse the range first.")
show(f"{station}: {start} → {end}, {len(labels)} labels of labeling/auto/{station}.jsonl, per family and per instant", n)
+31
View File
@@ -0,0 +1,31 @@
"""For the upgrader: the cadence of the measurements of the range, the number of targets and references, then each event
of the range with its sensors (categories.py only gives their number). Same computation as categories.py without
--chunk (3 days of history).
Usage: python3 detectors/scripts/learner/sensors.py <folder> --from YYYY-MM-DD --to YYYY-MM-DD
"""
import argparse
import statistics
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import categories # noqa: E402
from data import iso, load # noqa: E402
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("folder")
p.add_argument("--from", dest="start", required=True)
p.add_argument("--to", dest="end", required=True)
a = p.parse_args()
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
start, end = day(a.start), day(a.end)
batch = {s: k for s, ms in load(a.folder).items() if (k := [m for m in ms if start - timedelta(days=3) <= m["t"] < end])}
ts = sorted({m["t"] for ms in batch.values() for m in ms if m["t"] >= start})
steps = [(y - x).total_seconds() / 60 for x, y in zip(ts, ts[1:])]
print(f"cadence: {statistics.median(steps):.0f} min (median of the steps between measurements)" if steps else "cadence: no measurement")
refs = sum(ms[0]["pattern_type"] == "global_ref" for ms in batch.values())
print(f"sensors: {len(batch) - refs} targets, {refs} references")
for v in (v for v in categories.events(batch, end, a.folder) if v["end"] >= iso(start)) if batch else []:
print(v["start"], "→", v["end"], f"{v['category']:<20}", "|", v["episodes"][0]["label"], "|",
", ".join(sorted(v["sensors"])) or "all sensors")
+45
View File
@@ -0,0 +1,45 @@
"""Detector of the No data family (labeling/familles.md): moments without a measurement (the cadence of the "advanced"
export is broken).
Cadence: median gap between two measurement instants, all sensors together. A hole: more than MISSING missing steps
between two measurements (gap > MISSING + 1 cadences); the episode runs from the last measurement to the next one.
End of the window read (end, when the caller gives it: the chunked run of categories.py): the same rule between the last
measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of
the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the
export has nothing.
Usage: python3 detectors/no_data.py <station folder> (prints the episodes)
"""
import os
import statistics
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load # noqa: E402
MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23)
def duration(s):
h, m = divmod(round(s / 60), 60)
return f"{h} h {m:02d} min" if h else f"{m} min"
def detect(measurements, end=None):
"""end: end of the window read (datetime, excluded), optional; without it, a final hole is not seen."""
ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms})
if len(ts) < 3:
return []
gaps = [(b - a).total_seconds() for a, b in zip(ts, ts[1:])]
step = statistics.median(gaps)
out = [(a, b, g, "") for a, b, g in zip(ts, ts[1:], gaps)]
if end: # end excluded: the last expected measurement is at end − 1 step, the gap counts as between two measurements
out.append((ts[-1], end, (end - ts[-1]).total_seconds(), ", up to the end of the period"))
return [{"start": iso(a), "end": iso(b), "kind": "No data", "label": "No data",
"text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).",
"ignore": False, "sensors": []}
for a, b, g, tail in out if g > (MISSING + 1) * step]
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e)
+36
View File
@@ -0,0 +1,36 @@
"""Detector of the Obstruction family (labeling/familles.md): an object hides one or a few targets, for an instant or
for long. It reuses two computations shared with other families:
- the error_value of visibility.py: Brief obstruction (something passing, one instant) and Target obscured (a target
rejected again and again while the others stay seen, or a nuisance that only touches a few targets, no reference);
- the day-by-day rejections of target_issue.py: a target rejected for less than a week, Target obscured.
A sensor already in an overlapping Target issue episode (Degraded target, Target stuck) is not returned as Obstruction:
its curve is already to be ignored, the family is Target issue (boundary "Target issue or an episode of another family"
of labeling/familles.md; learner session Obstruction, mod. 1: S1, problem targets of FLORIDA, NORDEND, SNCF01,
NIEDERHORN, JUNGFRAU, CLARIDEN cut into Target obscured; S3, their isolated rejections as Brief obstruction).
Usage: python3 detectors/obstruction.py <station folder> (prints the episodes)
"""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it
import target_issue # noqa: E402
import visibility # noqa: E402
from data import load # noqa: E402
def detect(measurements):
ti = target_issue.episodes(measurements)
issues = [e for e in ti if e["kind"] == "Target issue"]
out = []
for e in visibility.episodes(measurements) + ti:
if e["kind"] != "Obstruction":
continue
taken = {s for h in issues if h["start"] <= e["end"] and h["end"] >= e["start"] for s in h["sensors"]}
if (e := {**e, "sensors": [s for s in e["sensors"] if s not in taken]})["sensors"]: # minus the sensors
out.append(e) # already in Target issue
return sorted(out, key=lambda e: e["start"])
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])
+184
View File
@@ -0,0 +1,184 @@
"""Detector of the Target issue family (labeling/familles.md): a persistently problematic target: rejected most of the
time, day after day, while the others are seen (degraded target), stuck (end of file), or flipping (the jumps of
displacement.py, "Target flipping"). A target rejected for less than a week is hidden at times (point 4): episodes()
returns it with the Obstruction family, which detect() leaves to obstruction.py.
visibility.py compares each sensor with its own usual level: a target that is almost always rejected looks normal to it
(FLORIDA, October 2026: Massif 2 and Sellette 6 rejected half the measurements or more, every day).
Method:
1. Calm instant for a sensor: at most CALM of the other sensors rejected (error_value > error_threshold); weather and
fog, which touch everyone, are thus set aside.
2. Per UTC day and per sensor, on its calm instants (at least MIN_HOURS hours of instants at the station's cadence,
median gap between two measurement instants; otherwise the day does not count): degraded day if at least SHARE of
them are rejected. Cadence (learner NEBRASKA, batch 01): 12 fixed instants, "3 h at 15 min", counted no day at one
measurement every 4 h; REFCOLH22, rejected all the time since it was lost, was not seen.
3. Episode: at least MIN_DAYS degraded days in a row (a day that does not count does not break the run), from the first
rejected measurement of these days to the last; ignore = True: the target's curve is not reliable.
4. Hindered all the time or at times (boundary "Target issue or Obstruction" of labeling/familles.md; learner
NEBRASKA, batch 01): a run of at least WEEK days is a problem target, "Degraded target"; shorter, a target hidden at
times, "Target obscured" (Obstruction family), one episode per run of rejected measurements (the hours when it is
seen are not part of it). BREITHORN: vehicles parked in front of COLI24 and POUI22, 2 to 5 days; NEBRASKA and
MICHIGAN: a lost reference, 25 days; JUNGFRAU: PM_G, 63.
For the problem target, the run is bridged: at most BRIDGE non-degraded days in a row do not break it, and it then
needs WEEK days from its first to its last degraded day, at SHARE_WEEK of rejections over all its days ("rejections
specific to the target almost every day of the range"); the Target obscured episodes it covers no longer exist
(learner session Obstruction, mod. 1, S1: FLORIDA Sellette 6 and Massif 2, SNCF01 F4.3 and F4.4, NORDEND CTR01,
NIEDERHORN e01, cut into Target obscured by a day under SHARE; BREITHORN, parked vehicles, unchanged).
The problem target judges regularity rather than the share of the day: a day is degraded from SHARE_WEEK (a quarter)
of its calm instants rejected, and BRIDGE is one day; the target hidden at times keeps SHARE. A target rejected every
day at the same hours, by day only or by night only, stays under half and was not seen (learner session Target
issue, mod. 1, S1: WILDSTRUBEL R3, 04:00 to 16:00 every day; NORDEND PR04 and CTR01, NIEDERHORN PWD_IMP_B08_024_C09
and e01, SNCF01 F4.3).
The cause (dirt, vegetation, setting) can only be read on the images.
Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per UTC day, the noise of its corr_x and
corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on
both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at
7e-5 px and C3 at 2e-4 against 0.013 for the others.
Usage: python3 detectors/target_issue.py <station folder> (prints the episodes)
"""
import os
import statistics
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it
import displacement # noqa: E402
from data import is_reference, iso, load # noqa: E402
CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant
SHARE = 0.5 # half of the calm instants rejected: "most of the time"
SHARE_WEEK = 0.25 # a quarter of the calm instants rejected, each day of a week or more: a problem target, even when
# rejected by day only or by night only (learner session Target issue, mod. 1, S1)
MIN_HOURS = 3 # hours of calm instants, at the station's cadence: below that, the day does not count
MIN_DAYS = 2 # a single day is too often an accident (NORDEND: isolated days on 16 sensors)
BRIDGE = 1 # non-degraded days that do not break a run of degraded days (4); at 2, the parked vehicles of BREITHORN
# (two days, then nothing for two days, then one) would make a week at a quarter
WEEK = 7 # degraded days in a row for a problem target; below, a target hidden at times
STUCK = 0.05 # noise of a stuck target compared with the others (E-011: 0.006; NORDEND, E-001: 0.7 at the lowest)
def is_rejected(m):
return m["error_value"] is not None and m["error_threshold"] is not None and m["error_value"] > m["error_threshold"]
def cadence(measurements):
"""Median gap between two measurement instants, all sensors together, in hours (as no_data)."""
ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms})
return statistics.median((b - a).total_seconds() for a, b in zip(ts, ts[1:])) / 3600 if len(ts) > 1 else 1
def episodes(measurements):
"""Targets rejected day after day: Target issue (a week or more) and Obstruction (less), then the stuck ones."""
min_instants = MIN_HOURS / cadence(measurements)
by_t = {}
for s, ms in measurements.items():
for m in ms:
by_t.setdefault(m["t"], {})[s] = is_rejected(m)
days = {} # sensor -> day -> [rejected, calm, first measurement, last]
for t in sorted(by_t):
d = by_t[t]
for s, r in d.items():
others = [v for k, v in d.items() if k != s]
c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t])
c[3] = t
if others and sum(others) / len(others) <= CALM:
c[0] += r
c[1] += 1
out = []
for s, ds in sorted(days.items()):
covered = [] # bridged runs of at least WEEK days, at SHARE_WEEK of rejections: a problem target
for run in _runs(ds, min_instants, BRIDGE, SHARE_WEEK):
share = sum(c[0] for c in run) / sum(c[1] for c in run)
if len(run) < WEEK or share < SHARE_WEEK:
continue
n = sum(c[0] / c[1] >= SHARE_WEEK for c in run)
r = _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])
covered.append((run[0][2], run[-1][3]))
out.append({"start": iso(r[0][0]), "end": iso(r[-1][1]), "kind": "Target issue", "label": "Degraded target",
"text": f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were "
f"seen, on {n} of {len(run)} days.", "ignore": True, "sensors": [s]})
for run in _runs(ds, min_instants, 0, SHARE): # at times: one episode per run of rejected measurements
if len(run) < MIN_DAYS or any(a <= run[0][2] <= b for a, b in covered):
continue
share = round(100 * sum(c[0] for c in run) / sum(c[1] for c in run))
text = f"{s} was rejected in {share} % of the measurements when the other targets were seen, {len(run)} days in a row."
out += [{"start": iso(a), "end": iso(b), "kind": "Obstruction", "label": "Target obscured", "text": text,
"ignore": True, "sensors": [s]}
for a, b in _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])]
return sorted(out + stuck(measurements, min_instants), key=lambda e: e["start"])
def _runs(ds, min_instants, bridge, share):
"""{day: [rejected, calm, first, last]} -> runs of degraded days; at most `bridge` non-degraded days in a row between
two degraded days do not break the run (they belong to it); a day that does not count does not break it."""
runs, cur, gap = [], [], []
for j in sorted(ds):
rej, calm = ds[j][:2]
if calm < min_instants:
continue
if rej / calm >= share:
cur += gap + [ds[j]]
gap = []
elif cur:
gap.append(ds[j])
if len(gap) > bridge:
runs.append(cur)
cur, gap = [], []
return runs + ([cur] if cur else [])
def _rejections(ms):
"""Measurements of a sensor -> [(start, end)] of its runs of rejected measurements."""
out, cur = [], []
for m in ms + [None]:
if m and is_rejected(m):
cur.append(m["t"])
elif cur:
out.append((cur[0], cur[-1]))
cur = []
return out
def _noise(v):
return statistics.median(abs(y - x) for x, y in zip(v, v[1:])) if len(v) > 2 else None
def stuck(measurements, min_instants):
"""Targets whose corr_x and corr_y barely move, day after day, while the others move."""
days = {} # day -> target -> (x noise, y noise, first measurement, last)
for s, ms in measurements.items():
if is_reference(ms):
continue
by_day = {}
for m in ms:
by_day.setdefault(m["t"].date(), []).append(m)
for d, dm in by_day.items():
b = [_noise([m[c] for m in dm if m[c] is not None]) for c in ("corr_x", "corr_y")]
if None not in b and len(dm) >= min_instants:
days.setdefault(d, {})[s] = (*b, dm[0]["t"], dm[-1]["t"])
runs = {} # target -> [[stuck days in a row]]
for d in sorted(days):
cs = days[d]
for s, (bx, by, a, b) in cs.items():
others = [v for k, (x, y, *_) in cs.items() if k != s for v in (x, y)]
if len(others) < 4:
continue
ref = statistics.median(others)
ru = runs.setdefault(s, [[]])
if ref and max(bx, by) <= STUCK * ref:
ru[-1].append((a, b))
elif ru[-1]:
ru.append([])
return [{"start": iso(k[0][0]), "end": iso(k[-1][1]), "kind": "Target issue", "label": "Target stuck",
"text": f"{s} barely moved while the other targets did ({len(k)} day{'s' if len(k) > 1 else ''}): it may be "
"set on a reference prism.", "ignore": True, "sensors": [s]}
for s, ru in sorted(runs.items()) for k in ru if k]
def detect(measurements):
return sorted((e for e in episodes(measurements) + displacement.episodes(measurements) if e["kind"] == "Target issue"),
key=lambda e: e["start"])
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e)
+292
View File
@@ -0,0 +1,292 @@
"""Detector of the Thermal displacement family (labeling/familles.md): the daily wave of the corrected displacements
(expansion by day, contraction in the evening).
Method, per target (global references excluded) and per axis (corr_x, corr_y):
1. Each UTC day of at least MIN_HOURS hours: mean of each hour minus the median of the day (removes jumps and drift).
2. Profile = median, hour by hour, over the days; "repeatability" = median, over the days, of the correlation between
the day and the profile of the other days. Axis concerned if repeatability ≥ REPRO. An incomplete day (at least
PARTIAL hours, fewer than MIN_HOURS) does not enter the profile, but its likeness is measured on the hours it has: a
hole of half a day no longer takes its episode away (learner session Thermal displacement, mod. 1, S4; at
MIN_HOURS = 12 or 16, the incomplete days distorted the profile, NIEDERHORN went back).
3. Cycle of the station if at least SHARE of the targets have an axis concerned, **and** (second method, asked by the
user) if a 24 h sinusoid, a + b·sin(2πt/24 h) + c·cos(2πt/24 h) by least squares, explains at least R2_MIN of the
curve of an axis for at least SHARE of the targets.
4. Warm hours: night level = median of the NIGHT flattest consecutive hours of the profile; an hour is warm if, as a
median over the axes concerned, the profile moves away from it by at least WARM times its amplitude.
5. Two episodes per complete day that looks like the others (median correlation ≥ DAY_MATCH over the axes concerned):
"Thermal expansion" from the start to the hour of the largest gap from the night level (the peak), then "Thermal
contraction" until the return. The warm hours say there is a cycle (at least MIN_WARM_HOURS of them); its bounds
are read lower, at EDGE times the amplitude: the start, going back from the peak while the gap stays ≥ EDGE (at most
to the calmest hour of the profile); the return, going down from the peak until it goes back under EDGE, after
midnight if needed, at most to the next day's start: a slow return at night chains the cycles (learner session
Thermal displacement, mod. 1, S1). The direction (a rising or falling curve) depends on the target's orientation:
it is not looked at, only the gap from the night level.
Reading: the cycle is almost everywhere in the raw positions (common mode) and the compensation removes it, except at
PS des Coudres where it stays in corr, with the same shape as in the raw positions.
Camera thermal effect (D-074): without a cycle in corr, the same method on the raw positions x, y gives "Camera thermal
expansion" and "Camera thermal contraction": the camera moves with the heat of its mount, the compensation removes it;
same categories, own sentence. On the raw positions the second method is not asked (WILDSTRUBEL, batch 01: the camera
rose by a 3 px block at sunrise, held 3 h and dropped in one step around 09:30, six days in a row, which the sinusoid
does not explain); in exchange, a single warm hour is not a cycle (MIN_WARM_HOURS: NIEDERHORN, E-005, hourly, would
otherwise have its noon hour as a camera contraction). The camera episode covers all sensors, references included: the
whole image moves.
Common slide, without a cycle (learner MICHIGAN, mod. 1): lacking a cycle in the corrected curves as in the raw
positions, an instant is a camera slide if the raw positions (x or y) of at least SHARE of the sensors, references
included, **and** the corrected curves (corr_x or corr_y) of at least SHARE of the targets move away on the same side by
at least SLIDE_K noises (median of the steps of the series) from their median of the day before; slides less than a
day apart make one episode, "Camera thermal movement", on all sensors, host of the jumps back (categories.HOSTS). The
condition on the raw positions leaves to the jumps a flip of the computed pose alone (raw positions still).
The camera moves with the cold (learner session Thermal displacement, mod. 1, S3): lacking a cycle, also a "Camera
thermal movement" where the camera pose (tilt, rotation or scale) moves away from its median of the day before by at
least COLD noises, while the station temperature (detect declares it, D-124) falls under its own by at least COLD
noises the same hour; noise: the median of the variations over an hour (over one step, at one measurement every 4 h),
so that the threshold does not depend on the cadence. The targets may move in opposite directions or stay almost still
(the compensation absorbs the movement): the slide rule does not see it. Consecutive pose instants make one episode.
Without a pose or without a temperature, nothing. The heat is not taken: in fine weather, it is the day's cycle.
"""
import math
import os
import statistics
import sys
from datetime import datetime, timedelta, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load # noqa: E402
MIN_HOURS = 20 # a UTC day counts in the profile if it has at least 20 measured hours
PARTIAL = 12 # an incomplete day of at least 12 hours gets its episode if it looks like the profile
REPRO = 0.6 # an axis has a cycle if each day looks like the others (pure noise ≈ 0)
SHARE = 0.5 # share of the targets concerned; 2026-10_NORDEND is just under (9/19)
WARM = 0.3 # gap from the night level, as a share of the profile's amplitude
DAY_MATCH = 0.5 # likeness of a day to the others to give it its episode
NIGHT = 8 # flattest consecutive hours of the profile = night level
R2_MIN = 0.3 # draft of 2026-10-24: PS des Coudres 14 axes out of 22 above, CLARIDEN 0 out of 48
MIN_WARM_HOURS = 2 # WILDSTRUBEL, batch 01; NIEDERHORN (E-005) has only one warm hour (noon)
EDGE = 0.1 # bounds of the wave (start, return), as a share of the amplitude; WARM says whether there is a cycle
SLIDE_K = 3 # common slide, in noises (median of the steps) away from the median of the day before
COLD = 4 # the camera moves with the cold, in one-hour noises away from the day before (pose, temperature)
W = 2 * math.pi / 86400 # angular frequency of a 24 h cycle, in rad/s
def _corr(a, b):
ma, mb = statistics.mean(a), statistics.mean(b)
num = sum((x - ma) * (y - mb) for x, y in zip(a, b))
den = (sum((x - ma) ** 2 for x in a) * sum((y - mb) ** 2 for y in b)) ** 0.5
return num / den if den else 0.0
def _axis(ms, col):
"""-> (24 h profile, {day: correlation with the profile of the other days}) or None."""
days = {}
for m in ms:
if m[col] is not None:
days.setdefault(m["t"].date(), {}).setdefault(m["t"].hour, []).append(m[col])
full, every = {}, {}
for d, hs in days.items():
if len(hs) >= PARTIAL:
med = statistics.median(v for l in hs.values() for v in l)
every[d] = {h: statistics.mean(l) - med for h, l in hs.items()}
if len(hs) >= MIN_HOURS:
full[d] = every[d]
if len(full) < 2:
return None
profile = [statistics.median(q[h] for q in full.values() if h in q) for h in range(24) if any(h in q for q in full.values())]
if len(profile) < 24:
return None
likeness = {}
for d, p in every.items():
others = [q for e, q in full.items() if e != d]
hs = [h for h in p if all(h in q for q in others)]
if len(hs) >= 3:
likeness[d] = _corr([p[h] for h in hs], [statistics.median(q[h] for q in others) for h in hs])
return profile, likeness
def sine_r2(ms, col):
"""Share of the curve explained by a 24 h sinusoid (least squares, 3 unknowns), or None if too short."""
pts = [(m["t"].timestamp(), m[col]) for m in ms if m[col] is not None]
if len(pts) <= 48:
return None
rows = [(1.0, math.sin(W * t), math.cos(W * t)) for t, _ in pts]
A = [[sum(r[i] * r[j] for r in rows) for j in range(3)] for i in range(3)]
B = [sum(r[i] * y for r, (_, y) in zip(rows, pts)) for i in range(3)]
for i in range(3): # Gaussian elimination
for k in range(i + 1, 3):
f = A[k][i] / A[i][i]
A[k] = [x - f * y for x, y in zip(A[k], A[i])]
B[k] -= f * B[i]
x = [0.0] * 3
for i in (2, 1, 0):
x[i] = (B[i] - sum(A[i][j] * x[j] for j in range(i + 1, 3))) / A[i][i]
m = statistics.mean(y for _, y in pts)
sst = sum((y - m) ** 2 for _, y in pts)
sse = sum((y - sum(c * v for c, v in zip(x, r))) ** 2 for r, (_, y) in zip(rows, pts))
return 1 - sse / sst if sst else None
def sine(measurements, cols=("corr_x", "corr_y")):
"""-> (targets with an axis following a 24 h sinusoid, targets evaluated)."""
yes, evaluated = set(), set()
for s, ms in measurements.items():
if ms[0]["pattern_type"] == "global_ref":
continue
for col in cols:
r2 = sine_r2(ms, col)
if r2 is not None:
evaluated.add(s)
if r2 >= R2_MIN:
yes.add(s)
return yes, evaluated
def cycle(measurements, cols=("corr_x", "corr_y")):
"""-> (axes concerned [(sensor, col, profile, likeness)], number of targets evaluated)."""
concerned, targets = [], set()
for s, ms in measurements.items():
if ms[0]["pattern_type"] == "global_ref":
continue
for col in cols:
r = _axis(ms, col)
if r is None:
continue
targets.add(s)
if statistics.median(r[1].values()) >= REPRO:
concerned.append((s, col) + r)
return concerned, len(targets)
def gaps(concerned):
"""Median gap (over the axes concerned) from the night level, hour by hour, as a share of the profile's amplitude."""
def score(p, h):
start = min(range(24), key=lambda i: max(p[(i + k) % 24] for k in range(NIGHT)) - min(p[(i + k) % 24] for k in range(NIGHT)))
night = statistics.median(p[(start + k) % 24] for k in range(NIGHT))
return abs(p[h] - night) / (max(p) - min(p))
return [statistics.median(score(c[2], h) for c in concerned) for h in range(24)]
def warm_hours(concerned):
return [h for h, e in enumerate(gaps(concerned)) if e >= WARM]
def detect(measurements, temperature=(), camera=()):
"""Cycle of the corrected curves (the structure); otherwise, that of the raw positions (the camera moving with the
expansion of its mount, compensated: D-074); otherwise, the camera slides (with the cold)."""
return (_episodes(measurements, ("corr_x", "corr_y"), "Thermal", False) or
_episodes(measurements, ("x", "y"), "Camera thermal", True) or slides(measurements, temperature, camera))
def _signs(ms, col, hourly=False, k=None):
"""{instant: +1 or -1} where the measurement moves away by at least k (default SLIDE_K) noises from the median of the
day before; noise: the median of the steps from one measurement to the next or, hourly, of the variations over an
hour (one step if the cadence is slow)."""
pts = [(m["t"], m[col]) for m in ms if m[col] is not None]
if len(pts) < 3:
return {}
step = statistics.median((b - a).total_seconds() for (a, _), (b, _) in zip(pts, pts[1:])) if hourly else 0
n = max(1, round(3600 / step)) if step else 1
noise = statistics.median(abs(pts[i + n][1] - pts[i][1]) for i in range(len(pts) - n)) if len(pts) > n else 0
out, j = {}, 0
for i, (t, v) in enumerate(pts):
while pts[j][0] < t - timedelta(days=1):
j += 1
before = [w for _, w in pts[j:i]]
if noise and len(before) >= 3 and abs(v - statistics.median(before)) >= (k or SLIDE_K) * noise:
out[t] = 1 if v > statistics.median(before) else -1
return out
def _common(series):
"""[{instant: sign}] of a group of sensors -> instants where at least SHARE of them move away on the same side."""
n, count = len(series), {}
for sg in series:
for t, x in sg.items():
count[(t, x)] = count.get((t, x), 0) + 1
return {t for (t, x), c in count.items() if n and c >= SHARE * n}
def _cold(measurements, temperature, camera):
"""[(start, end)] of the runs of instants where the camera pose (tilt, rotation or scale) moves away from its day
before while the temperature falls under its own, by at least COLD one-hour noises; empty without one of them."""
ts = [m["t"] for ms in measurements.values() for m in ms]
hour = lambda t: t.replace(minute=0, second=0, microsecond=0) # noqa: E731
cold = {hour(t) for t, x in _signs([{"t": t, "T": v} for t, v in temperature], "T", True, COLD).items() if x < 0}
pose = [p for p in camera if min(ts) <= p["t"] <= max(ts)] if ts else []
seen = {t for c in ("tilt", "rotation", "scale") for t in _signs(pose, c, True, COLD) if hour(t) in cold}
episodes = [] # consecutive pose instants make one episode
for i, p in enumerate(pose):
if p["t"] in seen:
if episodes and episodes[-1][2] == i - 1:
episodes[-1][1:] = [p["t"], i]
else:
episodes.append([p["t"], p["t"], i])
return [(a, b) for a, b, _ in episodes]
def slides(measurements, temperature=(), camera=()):
"""Common camera slide, without a 24 h cycle: the raw positions of most sensors, references included, and the
corrected curves of most targets move away together, on the same side, from their day before; or the camera pose
moves away from it with the cold (_cold)."""
targets = [ms for ms in measurements.values() if ms[0]["pattern_type"] != "global_ref"]
raw = set().union(*(_common([_signs(ms, c) for ms in measurements.values()]) for c in ("x", "y")))
corr = set().union(*(_common([_signs(ms, c) for ms in targets]) for c in ("corr_x", "corr_y")))
episodes = []
for t in sorted(raw & corr):
if episodes and t - episodes[-1][1] <= timedelta(days=1): # less than a day between two: the same slide
episodes[-1][1] = t
else:
episodes.append([t, t])
return [{"start": iso(a), "end": iso(b), "kind": "Thermal displacement", "label": "Camera thermal movement",
"text": "The camera moved with a change of temperature and all the curves moved with it; this is not a "
"movement of the structure, nothing to do.", "ignore": False, "sensors": sorted(measurements)}
for a, b in episodes + _cold(measurements, temperature, camera)]
def _episodes(measurements, cols, name, camera):
concerned, n = cycle(measurements, cols)
sensors = sorted({c[0] for c in concerned})
if not n or len(sensors) < SHARE * n:
return []
if not camera: # second method, for the structure only: the sinusoid must see it too
yes, evaluated = sine(measurements, cols)
if not evaluated or len(yes) < SHARE * len(evaluated):
return []
warm = warm_hours(concerned)
if len(warm) < MIN_WARM_HOURS:
return []
e = gaps(concerned)
peak = max(warm, key=lambda h: e[h]) # the largest gap from the night level: end of the expansion
rest = min(range(24), key=lambda h: e[h])
a = peak # start: going back from the peak while the profile stays above EDGE, up to the calmest hour
while (a - 1) % 24 != rest and e[(a - 1) % 24] >= EDGE:
a -= 1
b = peak + 1 # return: until the profile goes back under EDGE, at most to the next start
while b < a + 24 and e[b % 24] >= EDGE:
b += 1
last = max(m["t"] for ms in measurements.values() for m in ms)
days = sorted({d for c in concerned for d in c[3]})
if camera: # the whole image moves, references included
sensors = sorted(measurements)
episodes = []
for d in days: # ponytail: days of at least PARTIAL hours, the export's ends (22:00 → midnight) get no episode
if statistics.median(c[3][d] for c in concerned if d in c[3]) < DAY_MATCH:
continue
midnight = datetime(d.year, d.month, d.day, tzinfo=timezone.utc)
for label, u, v in ((f"{name} expansion", a, peak), (f"{name} contraction", peak, b)):
start, end = midnight + timedelta(hours=u), min(midnight + timedelta(hours=v), last)
if end > start:
episodes.append({"start": iso(start), "end": iso(end),
"kind": "Thermal displacement", "label": label, "text": "", "ignore": False, "sensors": sensors})
return episodes
if __name__ == "__main__":
measurements = load(sys.argv[1])
for e in detect(measurements):
print(e["start"], "→", e["end"], e["label"], len(e["sensors"]), "targets")
for cols in (("corr_x", "corr_y"), ("x", "y")):
concerned, n = cycle(measurements, cols)
k = len({c[0] for c in concerned})
amp = statistics.median(max(c[2]) - min(c[2]) for c in concerned) if concerned else 0
yes, evaluated = sine(measurements, cols)
print(f"{cols[0][:-2] or 'raw'}: targets {k}/{n}, median amplitude {amp:.3f} px, warm hours (UTC) "
f"{warm_hours(concerned) if concerned else '-'}; sinusoid: targets {len(yes)}/{len(evaluated)}")
+342
View File
@@ -0,0 +1,342 @@
"""Detector of the Visibility family (labeling/familles.md): the camera sees less well, the image is faint or blurred
(error_value), or it trembles (heat shimmer, end of file). The same error_value computation also finds objects in front
of a few targets (Brief obstruction, Target obscured): episodes() returns them with the Obstruction family, which
detect() leaves to obstruction.py.
Method:
1. "High" measurement: error_value > median + K_MAD MAD of the sensor over the export, and > FACTOR × median, or
rejected (> error_threshold) and > REJECT_FACTOR × median: a sensor at a usual level of 0.6 never doubles, and
FLORIDA (batch 01) showed its fogs as Target obscured.
2. Brief obstruction: an instant where at least BRIEF_MIN sensors are high, SALIENCE times more than at the neighbouring
instants (train, vehicle), with no reference (global_ref) among them: a one-step peak touching the references is not
an obstacle in front of targets (peak of the whole image, NORDEND on 01.09 11:15, batch 01); it is not shown. Unless
at least BRIEF_MIN targets are rejected at this instant and neither at the instant before nor after: something
passing in front of them, a reference touched as well (learner session Displacement, mod. 1, S5: trains of
NIEDERHORN, 12 and 13 targets rejected).
Also, something passing in front of one or two targets, or a single reference: at an instant, fewer than BRIEF_MIN
sensors rejected at more than FACTOR times their median (without K_MAD: by day, heat shimmer inflates the MAD), at
most one reference, none of them rejected at the instant before or after, and fewer than BRIEF_MIN high sensors
(learner session Obstruction, mod. 1, S3: vehicles, passers-by, insects in front of one or two targets of
WILDSTRUBEL, NIEDERHORN, LENZSPITZE; a reference hidden by a truck at JUNGFRAU, BREITHORN).
3. Poor visibility: runs of at least RUN_MIN high measurements per sensor, joined across sensors (at least 2), reduced
to the instants where at least FRAC of the episode's peak is high.
4. Target obscured: a target (not a global_ref) rejected (> error_threshold) and above K_TARGET MAD, at least TARGET_MIN
times less than TARGET_GAP apart, mostly outside the episodes of 2 and 3 (leaves, snow, glare on a target). The gap
is at least two measurement steps, and two rejections are enough when TARGET_MIN steps do not fit in TARGET_GAP
(learner session Obstruction, mod. 1, S6: at one measurement every 4 h, vehicles parked at night in front of COLI22,
COLI24, POUI22 of BREITHORN make only two or three rejections). All episodes: ignore = True.
5. Level of a Poor visibility (validated 2026-10-24), measurement by measurement: "none" (nothing is seen) when at least
HALF of the usually seen targets (under their threshold at least half the time) are rejected, "reduced" (average
data) otherwise; one episode per period of the same level, with the level field.
6. Bounds of a Poor visibility (WILDSTRUBEL, batch 01): from the first to the last measurement where a target is
rejected (> error_threshold), and no episode without a rejected target. On a road in full sun, the daytime
error_value (×1.5 to ×3, heat shimmer) made reduced visibility last for hours without any rejection; references do
not count: R3 is above its threshold almost all day.
7. Poor visibility or object (boundary "Visibility or Obstruction" of labeling/familles.md; BREITHORN, batch 01): a
nuisance touching at most FEW targets, and no reference, is not diffuse: it is an object in front of them (a parked
vehicle), a "Target obscured" episode of its whole duration, without a level.
8. Diffuse nuisance, with or without rejections (learner session Visibility, mod. 1, S2): at each instant, the median,
over the sensors, of the error_value in MAD above each one's median; at least RUN_MIN instants in a row at K_DIFFUSE
or more make a Poor visibility "reduced". A diffuse nuisance entirely within heat shimmer is the heat shimmer: no
episode (otherwise it absorbed the jump back of the day before, WILDSTRUBEL on 04.10). Finds the drops of
visibility of the whole scene without rejection at JUNGFRAU, FLORIDA, NORDEND; at 2, nights labelled thermal at
JUNGFRAU; at 3, half the gain.
A simpler version (share of targets rejected above their threshold, without comparing each sensor with its own level)
was tried on 2026-10-24 and dropped: worse on five data sets out of six (git history of Support-Agent,
detectors/archive/visibilite_simple.py).
"""
import os
import statistics
import sys
from datetime import timedelta
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load # noqa: E402
K_MAD = 10 # tuned on E-001 (fog: 11 to 57 MAD) and E-005 (rain, trains: 10 to 70 MAD)
FACTOR = 2 # drops sensors with a near-zero MAD (E-004: PWD_REF10 ×1.6 at 14 MAD, E-005)
RUN_MIN = 3 # high measurements in a row for poor visibility (E-001, E-005 rain)
BRIEF_MIN = 3 # tuned on E-005 (trains: 5 to 17 sensors; the one with 2 targets is missed)
K_TARGET = 5 # tuned on E-005 (leaves: 6 to 25 MAD, rejected)
TARGET_MIN = 3 # repeated rejections of a target
FRAC = 0.15 # cuts the tails of PR09 + CTR01 (NORDEND October) without cutting the cobweb (E-002)
GAP_STEPS = 2.5 # steps tolerated between two high measurements of the same run (a missing step, clock to the second)
BREAK = timedelta(hours=2) # lull separating two episodes (NORDEND October, night of 13 to 14.10)
SALIENCE = 3 # a train stands out at least 3 times above the neighbouring instants (E-005: 5 to 17 against 0 to 2;
# the cobweb of E-002 varies from 2 to 9 from one step to the next)
TARGET_GAP = timedelta(hours=6) # tuned on E-005 (leaves: 7 rejections in 12 h, up to 5 h apart)
REJECT_FACTOR = 1.2 # a rejection at 10 MAD also counts (FLORIDA, batch 01: levels of 0.6, ×2 out of reach); drops C24
# of E-009 (level 0.68 just under the threshold 0.7, rejected at ×1.04)
HALF = 0.5 # "nothing is seen"; tuned on NORDEND (09.10 23:00: 68 % rejected) and E-005 (rain: 24 to 33 %)
K_DIFFUSE = 2.5 # diffuse nuisance (8); learner session Visibility, mod. 1, S2
FEW = 3 # Poor visibility on at most 3 targets, no reference: an object, Target obscured (7); BREITHORN,
# batch 01 (vehicles parked in front of 2 or 3 targets out of 12)
def _step(measurements):
ts = sorted({m["t"] for v in measurements.values() for m in v})
return statistics.median(b - a for a, b in zip(ts, ts[1:]))
def _group(points, gap):
"""[(t, sensor)] sorted by t -> groups [(start, end, {sensors}, [t])] whose points are ≤ gap from each other."""
groups = []
for t, s in points:
if groups and t - groups[-1][1] <= gap:
g = groups[-1]
groups[-1] = (g[0], t, g[2] | {s}, g[3] + [t])
else:
groups.append((t, t, {s}, [t]))
return groups
def _group_intervals(intervals, gap):
"""[(start, end, sensor)] sorted -> [(start, end, {sensors})] of the intervals that overlap (within gap)."""
out = []
for a, b, s in intervals:
if out and a - out[-1][1] <= gap:
out[-1] = (out[-1][0], max(out[-1][1], b), out[-1][2] | {s})
else:
out.append((a, b, {s}))
return out
def _episode(start, end, label, text, sensors, level=None):
e = {"start": iso(start), "end": iso(end), "kind": "Visibility" if label == "Poor visibility" else "Obstruction",
"label": label, "text": text,
"ignore": level != "reduced", "sensors": sorted(sensors)} # reduced visibility: average data, not greyed
return {**e, "level": level} if level else e
def _levels(measurements, instants):
"""t -> "none" if at least HALF of the usually seen targets are rejected at t, "reduced" otherwise."""
ratios = []
for v in measurements.values():
r = {m["t"]: m["error_value"] / m["error_threshold"] for m in v if m["error_value"] is not None and m["error_threshold"]}
if r and statistics.median(r.values()) < 1:
ratios.append(r)
return {t: "none" if sum(r.get(t, 0) > 1 for r in ratios) >= HALF * max(1, sum(t in r for r in ratios)) else "reduced"
for t in instants}
def episodes(measurements):
"""All error_value episodes: Visibility (Poor visibility), Obstruction (Brief obstruction, Target obscured)."""
step = _step(measurements)
gap = GAP_STEPS * step
refs = {s for s, v in measurements.items() if v[0]["pattern_type"] == "global_ref"}
instants = sorted({m["t"] for v in measurements.values() for m in v})
high, rejections = {}, {} # sensor -> [t]
refused = set() # instants where a target (not a global_ref) is above its threshold
rejected_targets = {} # t -> {targets above their threshold}
strong, refusals = {}, {} # t -> {sensors rejected at more than FACTOR times their median}, {sensors rejected}
for s, v in measurements.items():
e = [m["error_value"] for m in v if m["error_value"] is not None]
if len(e) < 10:
continue
med = statistics.median(e)
mad = statistics.median(abs(x - med) for x in e) or 1e-9
for m in v:
x = m["error_value"]
if x is None:
continue
rejected = m["error_threshold"] and x > m["error_threshold"]
if rejected and s not in refs:
refused.add(m["t"])
rejected_targets.setdefault(m["t"], set()).add(s)
if x > med + K_MAD * mad and (x > FACTOR * med or rejected and x > REJECT_FACTOR * med):
high.setdefault(s, []).append(m["t"])
if rejected:
refusals.setdefault(m["t"], set()).add(s)
if x > FACTOR * med:
strong.setdefault(m["t"], set()).add(s)
if s not in refs and rejected and x > med + K_TARGET * mad:
rejections.setdefault(s, []).append(m["t"])
by_t = {} # t -> {high sensors}
for s, ts in high.items():
for t in ts:
by_t.setdefault(t, set()).add(s)
n = lambda t: len(by_t.get(t, ())) # noqa: E731
out = []
# 2. Brief obstruction: an instant where many sensors are high, at least SALIENCE times more than the neighbours.
briefs = set()
for i, t in enumerate(instants):
neighbours = [n(instants[j]) for j in (i - 1, i + 1) if 0 <= j < len(instants)]
if n(t) >= BRIEF_MIN and n(t) >= SALIENCE * max(neighbours):
briefs.add(t)
only = rejected_targets.get(t, set()).difference(*(rejected_targets.get(instants[j], set()) for j in (i - 1, i + 1)
if 0 <= j < len(instants)))
if by_t[t] & refs and len(only) < BRIEF_MIN: # references touched without targets rejected at this instant
continue # alone: not an obstacle in front of targets, the whole image
out.append(_episode(t, t, "Brief obstruction", f"{n(t)} points were poorly seen for a single measurement, then came "
"back: probably a train, a vehicle or an animal passing; ignore this point on the curves.", by_t[t]))
elif 0 < len(f := strong.get(t, set())) < BRIEF_MIN and n(t) < BRIEF_MIN and len(f & refs) <= 1 and not f & set().union(
*(refusals.get(instants[j], set()) for j in (i - 1, i + 1) if 0 <= j < len(instants))):
# one or two targets, or a reference, rejected at this single measurement, at FACTOR times their median
out.append(_episode(t, t, "Brief obstruction", f"{len(f)} point{'s were' if len(f) > 1 else ' was'} poorly seen "
"for a single measurement, then came back: probably something passing; ignore this point on the "
"curves.", f))
# 3. Poor visibility: long runs per sensor (without the brief instants), joined across sensors, then reduced to the
# instants where at least FRAC of the episode's peak is high (cuts the tails of one or two sensors and the long
# lulls, more than BREAK).
runs = []
for s, ts in high.items():
pts = [(t, s) for t in ts if t not in briefs]
runs += [(a, b, s) for a, b, _, l in _group(pts, gap) if len(l) >= RUN_MIN]
zones = [z for z in _group_intervals(sorted(runs), gap) if len(z[2]) >= 2]
kept = []
for a, b, _ in zones:
inside = [t for t in instants if a <= t <= b and t not in briefs]
peak = max(n(t) for t in inside)
pts = [(t, s) for t in inside if n(t) >= max(1, FRAC * peak) for s in by_t[t]]
kept += [(d, f, cs) for d, f, cs, l in _group(pts, max(gap, BREAK)) if len(cs) >= 2 and len(set(l)) >= RUN_MIN]
# Bounds: from the first to the last rejected target; without a rejection, no episode (6).
kept = [(r[0], r[-1], cs) for a, b, cs in kept if (r := [t for t in instants if a <= t <= b and t in refused])]
level = _levels(measurements, instants)
for a, b, cs in kept: # one episode per period of the same level (5); the text comes from categories.py
if len(cs) <= FEW and not cs & refs: # an object in front of a few targets, not a diffuse nuisance (7)
out.append(_episode(a, b, "Target obscured", "", cs))
continue
seg = []
for t in [t for t in instants if a <= t <= b] + [None]:
if seg and (t is None or level[t] != level[seg[0]]):
who = cs & set().union(*(by_t.get(u, set()) for u in seg)) or cs
out.append(_episode(seg[0], seg[-1], "Poor visibility", "", who, level[seg[0]]))
seg = []
if t is not None:
seg.append(t)
inside_any = lambda t: t in briefs or any(a <= t <= b for a, b, _ in kept) # noqa: E731
# 4. Target obscured: a target rejected again and again, mostly outside the episodes above.
targets = []
for s, ts in rejections.items():
for a, b, _, l in _group([(t, s) for t in ts], max(TARGET_GAP, 2 * step)):
if len(l) >= (TARGET_MIN if TARGET_MIN * step <= TARGET_GAP else 2) and sum(map(inside_any, l)) * 2 < len(l):
targets.append((a, b, s))
for a, b, cs in _group_intervals(sorted(targets), timedelta(0)):
who, it, its = ("One target was", "it", "its") if len(cs) == 1 else (f"{len(cs)} targets were", "them", "their")
out.append(_episode(a, b, "Target obscured", f"{who} repeatedly lost while the targets around stayed clear: probably "
f"something in front of {it} (leaves, snow) or glare; {its} displacements in this period are unreliable.", cs))
return sorted(out, key=lambda e: e["start"])
def diffuse(measurements):
"""Diffuse nuisance (8): the runs of at least RUN_MIN instants where the median, over the sensors, of the error_value
in MAD above each one's median reaches K_DIFFUSE, with or without rejections -> Poor visibility "reduced"."""
z = {} # t -> {sensor: gap in MAD}
for s, v in measurements.items():
e = [m["error_value"] for m in v if m["error_value"] is not None]
if len(e) < 10:
continue
med = statistics.median(e)
mad = statistics.median(abs(x - med) for x in e) or 1e-9
for m in v:
if m["error_value"] is not None:
z.setdefault(m["t"], {})[s] = (m["error_value"] - med) / mad
highs = [(t, None) for t in sorted(z) if statistics.median(z[t].values()) >= K_DIFFUSE]
return [_episode(a, b, "Poor visibility", "", {s for t in l for s, x in z[t].items() if x >= K_DIFFUSE}, "reduced")
for a, b, _, l in _group(highs, GAP_STEPS * _step(measurements)) if len(l) >= RUN_MIN]
def detect(measurements):
shimmer = heat_shimmer(measurements) # a diffuse nuisance during heat shimmer is the heat shimmer (8)
return ([e for e in episodes(measurements) if e["kind"] == "Visibility"] + shimmer
+ [e for e in diffuse(measurements) if not any(t["start"] <= e["start"] and e["end"] <= t["end"] for t in shimmer)])
# --- Heat shimmer ----------------------------------------------------------------------------------------------------
# Atmospheric turbulence (fast noise on the corrected displacements). Method:
# 1. Per sensor and per hour H, on the measurements of [H − 1 h, H + 2 h): agitation = median of |Δcorr_x| + |Δcorr_y|
# between successive measurements (robust to one-step jumps), and median of the error_value.
# 2. Each is divided by the sensor's calm: the lower quartile of its hourly values over the export (often the night).
# 3. Turbulent hour: at least SHARE of the sensors at ≥ RATIO times their calm agitation, and median error_value of the
# sensors at ≥ RATIO_ERR times the calm (warm air blurs the spots: E-003).
# 4. Neighbouring hours merged (a one-hour hole tolerated), episode of at least MIN_DURATION, dropped if more than
# REJECT_MAX of the measurements are rejected beyond their sensor's habit (its share of rejections over the export):
# it is then poor visibility, another rule. A target that is always rejected (Target issue) does not count (learner
# session Visibility, mod. 1, S1: WILDSTRUBEL, R3 rejected all day, the days of heat shimmer were dropped).
# Greyed if the median agitation ≥ STRONG.
RATIO = 1.5 # tuned on E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night)
SHARE = 0.5 # share of the sensors agitated at the same time
RATIO_ERR = 1.5 # tuned on E-003 (error_value ×1.7 to ×3.2 the night; ≈ ×1 at noon elsewhere)
REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; rejections beyond the habit since learner
# session Visibility, mod. 1, S1 (WILDSTRUBEL: 4 to 14 %, against 9 to 22 % raw)
STRONG = 2.0 # E-003 (×2.3 to ×2.6) is strong; an episode between ×1.5 and ×2 stays visible
MIN_DURATION = timedelta(hours=3) # E-003 lasts 10-11 h; drops the morning of the real movement of E-004 (2 h), which
# the classification drops too, whatever the duration
HOUR = timedelta(hours=1)
def _hour(t):
return t.replace(minute=0, second=0, microsecond=0)
def _ratios(points):
"""[(t, value)] -> {hour H: median over [H − 1 h, H + 2 h) / lower quartile of these medians}."""
by = {}
for t, v in points:
for k in (-1, 0, 1):
by.setdefault(_hour(t) + k * HOUR, []).append(v)
med = {h: statistics.median(v) for h, v in by.items() if len(v) >= 3}
if len(med) < 8:
return {}
calm = statistics.quantiles(med.values(), n=4)[0]
return {h: v / calm for h, v in med.items()} if calm > 0 else {}
def hours(measurements):
"""-> {hour: {sensor: (agitation / calm, error_value / calm)}}."""
out = {}
for s, ms in measurements.items():
ag = _ratios([(b["t"], abs(b["corr_x"] - a["corr_x"]) + abs(b["corr_y"] - a["corr_y"]))
for a, b in zip(ms, ms[1:]) if a["corr_x"] is not None and b["corr_x"] is not None])
er = _ratios([(m["t"], m["error_value"]) for m in ms if m["error_value"] is not None])
for h, r in ag.items():
if h in er:
out.setdefault(h, {})[s] = (r, er[h])
return out
def _turbulent(r):
return (sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
and statistics.median(e for _, e in r.values()) >= RATIO_ERR)
def heat_shimmer(measurements):
H = hours(measurements)
blocks = []
for h in sorted(h for h, r in H.items() if _turbulent(r)):
if blocks and h - blocks[-1][-1] <= 2 * HOUR:
blocks[-1].append(h)
else:
blocks.append([h])
habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()} # usual rejections
episodes = []
for b in blocks:
start, end = b[0], b[-1] + HOUR
by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()}
new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items())
if end - start < MIN_DURATION or new > REJECT_MAX * sum(map(len, by.values())):
continue
by_sensor = {}
for h in b:
for s, (a, _) in H[h].items():
by_sensor.setdefault(s, []).append(a)
agit = {s: statistics.median(v) for s, v in by_sensor.items()}
strength = statistics.median(agit.values())
episodes.append({
"start": iso(start), "end": iso(end), "kind": "Visibility",
"label": "Heat shimmer",
"text": f"Readings were about {strength:.1f}× noisier than in calm hours because of heat shimmer "
"(warm air trembling in the line of sight); the small fast wiggles are not movement.",
"ignore": strength >= STRONG,
"sensors": sorted(s for s, v in agit.items() if v >= RATIO),
})
return episodes
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])