Detectors in English, under scripts/; learner tools under scripts/learner/
- data.py (was donnees.py): unused filtrer, heure_image and image sorting removed - score.py: dead functions (validite, marquer, detectes, score, familles) and the validity levels removed; --from/--to - regression.py: history.csv and --seance/--etape removed - categories.py: --from/--to/--chunk/--history; process values images/none/check - benchmark on all stations unchanged (same 132 figures) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9b3ac0ecfe
commit
969637499f
23 files changed
+1877
-1971
No files matched your search
@@ -0,0 +1,243 @@
|
||||
"""From the detectors' episodes to events, one per family.
|
||||
|
||||
One detector per label family (labeling/familles.md): visibility.py, obstruction.py, target_issue.py,
|
||||
thermal_displacement.py, no_data.py, displacement.py. The category of an event is its family, the kind field of its
|
||||
episodes; their label gives the variety (Poor visibility, Heat shimmer, Brief obstruction, Slow drift…). Rules:
|
||||
- a jump back to normal (JUMP_BACK) that overlaps an episode of HOSTS (loss of visibility, something passing, heat
|
||||
shimmer, and last the thermal effect, if it touches several targets) is absorbed into it: it takes its cause and
|
||||
family; otherwise it stays Displacement, cause not established, to be checked on the images (familles.md, "A jump then
|
||||
the return"); a Brief obstruction (one instant) only absorbs the jumps starting at its instant, not those of several
|
||||
days it crosses (learner session Displacement, mod. 1, S5);
|
||||
- a slow drift or a jump back whose targets are all in an overlapping Target issue episode is absorbed into it (the
|
||||
curve of a problem target is to be ignored, familles.md);
|
||||
- a shift that lasts, or too recent to know (SHIFTS), is never absorbed;
|
||||
- heat shimmer overlapping a shift of several targets is not kept: a real movement also shakes the curves; the shift of
|
||||
a single target does not remove the heat shimmer of the whole scene, and heat shimmer only absorbs the jumps back
|
||||
starting during it, not the one of the day before coming back within its day (learner session Visibility, mod. 1,
|
||||
S1: WILDSTRUBEL);
|
||||
- two events of the same variety less than MERGE apart make one; two Obstruction (Target obscured, Brief obstruction)
|
||||
only if they share a sensor (learner session Obstruction, mod. 1: S1, different targets chained into an event of
|
||||
several days; S3, passings of a few minutes into a whole hour); two Target issue too: each problem target keeps its
|
||||
bounds (learner session Target issue, mod. 1, S3); a Brief obstruction only absorbs the jumps back within MERGE after
|
||||
its instant (S3: not the 30 h jump of another target).
|
||||
Each event gets its sentence, in English (the site): what happened and what to do about it, as the "For the client"
|
||||
column of familles.md says; the cause is only given if the variety establishes it, otherwise the images show it.
|
||||
No detect function: the loader does not take this file for a detector.
|
||||
Usage: python3 detectors/scripts/categories.py <station folder> (one line per event: family, variety, sensors)
|
||||
... <folder> [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--chunk 3] [--history N] (batch run: each chunk of N days
|
||||
classified alone, with the days of history before it if they exist; only the events touching the chunk are
|
||||
shown; "nothing" for a quiet chunk)
|
||||
"""
|
||||
import functools
|
||||
import importlib.util
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
|
||||
from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402
|
||||
|
||||
# What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134):
|
||||
# temperature [(t, °C)], hourly median of the components; camera [pose] (pan, tilt, rotation, scale, sigma0);
|
||||
# metrics {name: [(t, value)]}, all numeric metrics (sensor_CPU_deg_c, sensor_GPU_deg_c, BusVoltage…).
|
||||
CONTEXT = {"temperature": load_temperature, "camera": load_camera, "metrics": load_metrics}
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _context(folder, key):
|
||||
return CONTEXT[key](folder)
|
||||
|
||||
|
||||
# The families of labeling/familles.md, in the order of the timeline rows; ignore: greyed zone (None: as its episodes
|
||||
# say: a total loss of visibility, strong heat shimmer); process: Images button, nothing, or Images and a link to
|
||||
# Diagnosis.
|
||||
CATEGORIES = {
|
||||
"Visibility": {"ignore": None, "process": "images"},
|
||||
"Obstruction": {"ignore": True, "process": "images"},
|
||||
"Target issue": {"ignore": True, "process": "images"},
|
||||
"Thermal displacement": {"ignore": False, "process": "none"},
|
||||
"No data": {"ignore": False, "process": "none"},
|
||||
"Displacement": {"ignore": False, "process": "check"},
|
||||
}
|
||||
JUMP_BACK = "Sudden jump, back to normal" # a jump back: its cause, if it overlaps a host; otherwise Displacement
|
||||
SHIFTS = {"Lasting shift", "Jump, not back yet", "Large shift"} # to be checked: never absorbed
|
||||
HOSTS = {"Poor visibility", "Brief obstruction", "Heat shimmer", "Thermal expansion", "Thermal contraction",
|
||||
"Camera thermal expansion", "Camera thermal contraction", "Camera thermal movement"} # give the cause of an
|
||||
# overlapping jump back
|
||||
MERGE = timedelta(hours=1) # same variety less than an hour apart: a single event (validated rule, 2026-10-24)
|
||||
|
||||
|
||||
def detectors():
|
||||
"""Modules of the *.py next to this file that have detect; a file that does not load is skipped with a message."""
|
||||
mods = []
|
||||
for path in sorted(Path(__file__).resolve().parent.glob("*.py")):
|
||||
if path == Path(__file__).resolve():
|
||||
continue
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location(f"detector_{path.stem}", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
except Exception as e: # noqa: BLE001 (a broken detector does not block the page)
|
||||
print(f"Detectors: {path.name} skipped ({type(e).__name__}: {e})", file=sys.stderr)
|
||||
continue
|
||||
if hasattr(mod, "detect"):
|
||||
mods.append(mod)
|
||||
return mods
|
||||
|
||||
|
||||
def variety(ep):
|
||||
"""The variety of an episode within its family: shifts together, the thermal effect without "Camera", the loss of
|
||||
visibility by level."""
|
||||
label = "shift" if ep["label"] in SHIFTS else ep["label"].removeprefix("Camera ").capitalize()
|
||||
return ep["kind"], label, ep.get("level")
|
||||
|
||||
|
||||
def is_thermal(ep):
|
||||
return ep["kind"] == "Thermal displacement"
|
||||
|
||||
|
||||
def overlap(a, b):
|
||||
return a["start"] <= b["end"] and a["end"] >= b["start"] # data.iso times: text order is time order
|
||||
|
||||
|
||||
def targets(n):
|
||||
return "one target" if n == 1 else f"{n} targets"
|
||||
|
||||
|
||||
def sentence(group, sensors, references):
|
||||
"""Sentence of an event (validated by the user on 2026-10-24), from the variety of its main episode; an absorbed
|
||||
jump adds that it has the same cause."""
|
||||
n, one, ep = len(sensors), len(sensors) == 1, group[0]
|
||||
jumps = [e for e in group[1:] if e["label"] == JUMP_BACK]
|
||||
jumped = len({s for e in jumps for s in e["sensors"]})
|
||||
lasting = len({s for e in group if e["label"] in SHIFTS for s in e.get("lasting", [])})
|
||||
refs = ", including the reference targets" if references & set(sensors) else ""
|
||||
camera = ep["label"].startswith("Camera")
|
||||
large = any(e["label"] == "Large shift" for e in group)
|
||||
sort = "shift" if ep["label"] in SHIFTS else f"{ep['label']} {ep['level']}" if ep.get("level") else ep["label"]
|
||||
text = {
|
||||
"Poor visibility none": f"The camera could not see {targets(n)} clearly{refs}. "
|
||||
"The curves in this period are not reliable: ignore them.",
|
||||
"Poor visibility reduced": f"The camera saw {targets(n)} less clearly than usual{refs}. "
|
||||
"The data in this period are less precise: use them with care.",
|
||||
"Brief obstruction": f"Something passed in front of {targets(n)} for a single measurement. Ignore this point on the curves.",
|
||||
"Target obscured": f"{', '.join(sensors)} {'was' if one else 'were'} hidden several times while the targets around "
|
||||
f"{'it' if one else 'them'} stayed clear. Ignore {'its curve' if one else 'their curves'} in this period.",
|
||||
"Heat shimmer": "Warm air made the readings shake. The small fast wiggles are not movement.",
|
||||
"expansion": ("The camera support expands in the day heat; the curves are corrected for it."
|
||||
if camera else "The structure expands in the day heat.") + " This daily wave is normal: nothing to do.",
|
||||
"contraction": ("The camera support contracts as it cools down; the curves are corrected for it."
|
||||
if camera else "The structure contracts as it cools down.") + " This daily wave is normal: nothing to do.",
|
||||
JUMP_BACK: f"{targets(n).capitalize()} jumped and came back. The cause is not established: look at the images.",
|
||||
"shift": (f"{targets(n).capitalize()}, most of the targets, shifted together: everything moved." if large
|
||||
else f"{targets(n).capitalize()} shifted and {'has' if one else 'have'} not come back yet." if not lasting
|
||||
else f"{targets(n).capitalize()} shifted and stayed shifted." if lasting == n
|
||||
else f"{targets(n).capitalize()} shifted; {lasting} of them stayed shifted.")
|
||||
+ " Check whether the structure moved: look at the images.",
|
||||
}.get(sort.split()[-1] if is_thermal(ep) else sort, ep["text"]) # No data, Target issue, Slow drift: the detector's text
|
||||
if ep["kind"] == "Target issue":
|
||||
text += f" Ignore {'its curve' if one else 'their curves'} in this period and look at the images of the target."
|
||||
if not jumped:
|
||||
return text
|
||||
return text + (f" The jump of {targets(jumped)} in this period has the same cause." if len(jumps) == 1
|
||||
else f" The jumps of {targets(jumped)} in this period have the same cause.")
|
||||
|
||||
|
||||
def event(group, references):
|
||||
"""Episodes of the same event, the main one first -> event."""
|
||||
cat, var = group[0]["kind"], variety(group[0])
|
||||
c = CATEGORIES.get(cat, {"ignore": None, "process": "none"})
|
||||
sensors = [] if any(not e["sensors"] for e in group) else list(dict.fromkeys(s for e in group for s in e["sensors"]))
|
||||
main = [e for e in group if variety(e) == var] # the duration is the main episodes', not the absorbed jumps'
|
||||
return {"start": min(e["start"] for e in main), "end": max(e["end"] for e in main), "category": cat,
|
||||
"label": cat, "text": sentence(group, sensors, references),
|
||||
"ignore": c["ignore"] if c["ignore"] is not None else any(e["ignore"] for e in main),
|
||||
"sensors": sensors, "process": c["process"], "episodes": group}
|
||||
|
||||
|
||||
def classify(episodes, references=frozenset()):
|
||||
"""Episodes of all detectors -> events sorted by start (rules: module docstring); references: the global_ref
|
||||
sensors, for the sentence."""
|
||||
eps = sorted(episodes, key=lambda e: e["start"])
|
||||
# a real movement is never taken for heat shimmer; the shift of a single target is not a movement of the scene
|
||||
checks = [e for e in eps if e["label"] in SHIFTS and len(e["sensors"]) > 1]
|
||||
eps = [e for e in eps if e["label"] != "Heat shimmer" or not any(overlap(e, c) for c in checks)]
|
||||
# the thermal wave absorbs last (a jump of something passing or of rain goes to them), and only the jumps of several
|
||||
# targets: the wave moves them together, a single target that jumps is not the wave (C2 of E-003)
|
||||
hosts = sorted((e for e in eps if e["label"] in HOSTS), key=is_thermal)
|
||||
host = {id(e): next((h for h in hosts if overlap(e, h) and (len(e["sensors"]) > 1 or not is_thermal(h))
|
||||
and (h["label"] != "Heat shimmer" or h["start"] <= e["start"])
|
||||
and (h["label"] != "Brief obstruction" or e["start"] == h["start"]
|
||||
and datetime.fromisoformat(e["end"]) - datetime.fromisoformat(h["start"]) <= MERGE)), None)
|
||||
for e in eps if e["label"] == JUMP_BACK}
|
||||
# a drift or a jump back whose targets are all in an overlapping Target issue episode: its curve is to be ignored,
|
||||
# it takes its family (learner session Displacement, mod. 1, S6)
|
||||
host.update({id(e): h for e in eps if e["label"] in (JUMP_BACK, "Slow drift") and not host.get(id(e))
|
||||
and (h := next((h for h in eps if h["kind"] == "Target issue" and overlap(e, h)
|
||||
and set(e["sensors"]) <= set(h["sensors"])), None))})
|
||||
groups = {id(e): [e] for e in eps if not host.get(id(e))}
|
||||
for e in eps:
|
||||
if host.get(id(e)):
|
||||
groups[id(host[id(e)])].append(e)
|
||||
merged = []
|
||||
for g in groups.values():
|
||||
prev = next((m for m in reversed(merged) if variety(m[0]) == variety(g[0]) and (g[0]["kind"] not in ("Obstruction", "Target issue")
|
||||
or set(g[0]["sensors"]) & {s for e in m for s in e["sensors"]})), None)
|
||||
end = max((e["end"] for e in prev if variety(e) == variety(g[0])), default=None) if prev else None
|
||||
if end and datetime.fromisoformat(g[0]["start"]) - datetime.fromisoformat(end) < MERGE:
|
||||
prev += g
|
||||
else:
|
||||
merged.append(g)
|
||||
return sorted((event(g, references) for g in merged), key=lambda v: v["start"])
|
||||
|
||||
|
||||
def events(measurements, end=None, folder=None):
|
||||
"""Measurements (data.load) -> events of all detectors: what the page receives. A detector also receives what its
|
||||
detect function declares: end, the end of the window read (batch run), for a hole touching it; a CONTEXT key, read
|
||||
from the station folder (once per folder), empty without a folder."""
|
||||
episodes = []
|
||||
for mod in detectors():
|
||||
try:
|
||||
params = inspect.signature(mod.detect).parameters
|
||||
kw = {k: _context(folder, k) if folder else {} if k == "metrics" else [] for k in CONTEXT if k in params}
|
||||
if "end" in params:
|
||||
kw["end"] = end
|
||||
episodes += mod.detect(measurements, **kw)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"Detectors: {mod.__name__} skipped ({type(e).__name__}: {e})", file=sys.stderr)
|
||||
return classify(episodes, {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="One line per event; --chunk: each chunk classified alone.")
|
||||
p.add_argument("folder")
|
||||
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD (default: start of the data)")
|
||||
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD (default: end of the data)")
|
||||
p.add_argument("--chunk", type=int, default=0, help="days per chunk (default: a single pass)")
|
||||
# without history, a chunk starting just before a real jump takes it for a jump back (PR17 at NORDEND, 08/10 02:30:
|
||||
# absorbed into the fog); 3 days give it back; user's rule (2026-10-24): 3 days of history by default, in the
|
||||
# learner as in the real application
|
||||
p.add_argument("--history", type=int, default=3, help="days of earlier measurements given to the detectors (default 3)")
|
||||
a = p.parse_args()
|
||||
measurements = load(a.folder)
|
||||
ts = [m["t"] for ms in measurements.values() for m in ms]
|
||||
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
|
||||
start = day(a.start) if a.start else min(ts).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
end = day(a.end) if a.end else max(ts) + timedelta(seconds=1)
|
||||
t = start
|
||||
while t < end:
|
||||
b = min(t + timedelta(days=a.chunk), end) if a.chunk else end
|
||||
h = t - timedelta(days=a.history)
|
||||
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
|
||||
evs = [v for v in events(chunk, b, a.folder) if v["end"] >= iso(t)] if chunk else []
|
||||
if a.chunk:
|
||||
print(f"== {iso(t)} → {iso(b)}: {len(evs) or 'nothing'}")
|
||||
for v in evs:
|
||||
print(v["start"], "→", v["end"], f"{v['category']:<20}", "greyed" if v["ignore"] else "kept",
|
||||
f"| {v['episodes'][0]['label']}", # the variety: the rule of the family's detector
|
||||
f"| {len(v['sensors']) or 'all'} sensors", "| absorbed:", ", ".join(
|
||||
f"{e['label']} {e['start'][5:16]}" for e in v["episodes"][1:]) or "—")
|
||||
t = b
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""Reading a station folder (standard library only). Every time is UTC.
|
||||
|
||||
Column formats: labeling/connaissances/colonnes-des-deplacements.md in Diagnostic-app.
|
||||
"""
|
||||
import csv
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Numeric columns of the "advanced" export; an empty cell becomes None.
|
||||
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
|
||||
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
|
||||
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
|
||||
# Camera pose (camera/): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
|
||||
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
|
||||
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
|
||||
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
|
||||
|
||||
|
||||
def _header(path):
|
||||
with open(path, newline="") as f:
|
||||
return next(csv.reader(f), [])
|
||||
|
||||
|
||||
def find_files(folder):
|
||||
"""Station folder -> {'advanced': [...], 'camera': [...]}: sorted CSV paths, subfolders included."""
|
||||
out = {"advanced": [], "camera": []}
|
||||
for p in sorted(glob.glob(os.path.join(folder, "**", "*.csv"), recursive=True)):
|
||||
header = _header(p)
|
||||
if "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
|
||||
out["advanced"].append(p)
|
||||
elif "median_pan" in header and "timestamp" not in header: # timestamp: the hourly export
|
||||
out["camera"].append(p)
|
||||
return out
|
||||
|
||||
|
||||
def _number(v):
|
||||
v = v.strip()
|
||||
return float(v) if v else None
|
||||
|
||||
|
||||
def load(folder):
|
||||
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
|
||||
|
||||
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
|
||||
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
|
||||
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
|
||||
"""
|
||||
by = {}
|
||||
for path in find_files(folder)["advanced"]:
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
|
||||
m["t"] = parse_time(r["time_utc"])
|
||||
m["pattern_type"] = r["pattern_type"]
|
||||
if rejected(m):
|
||||
m["corr_x"] = m["corr_y"] = None
|
||||
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
|
||||
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
|
||||
|
||||
|
||||
def load_temperature(folder):
|
||||
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
|
||||
hour; [] without these metrics. The components rise and fall together; the median drops the spike of a single
|
||||
one, the computing GPU (D-124)."""
|
||||
by = {}
|
||||
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
if (v := _number(r.get("metric") or "")) is not None:
|
||||
by.setdefault(parse_time(r["time_utc"]), []).append(v)
|
||||
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
|
||||
|
||||
|
||||
def load_camera(folder):
|
||||
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
|
||||
|
||||
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
|
||||
A measurement instant without a pose: the camera was not registered.
|
||||
"""
|
||||
by = {}
|
||||
for path in find_files(folder)["camera"]:
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
|
||||
p["t"] = parse_time(r["time_utc"])
|
||||
by[p["t"]] = p
|
||||
return [by[t] for t in sorted(by)]
|
||||
|
||||
|
||||
def load_metrics(folder):
|
||||
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
|
||||
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
|
||||
read. Name: the file's, without the station or the date."""
|
||||
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
|
||||
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
|
||||
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
|
||||
if not (m := METRIC_RE.match(os.path.basename(path))):
|
||||
continue
|
||||
name = m.group(1).removeprefix(station)
|
||||
if IDENTIFIERS.match(name):
|
||||
continue
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
try:
|
||||
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
|
||||
except ValueError:
|
||||
continue
|
||||
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
|
||||
return {n: sorted(d.items()) for n, d in sorted(by.items())}
|
||||
|
||||
|
||||
def is_reference(measurements):
|
||||
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
|
||||
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
|
||||
none, filled reproj_err or inlier (references only)."""
|
||||
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
|
||||
if t is not None:
|
||||
return t == "global_ref"
|
||||
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
|
||||
|
||||
|
||||
def rejected(m):
|
||||
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
|
||||
two values, an empty corr_x."""
|
||||
if m["error_value"] is not None and m["error_threshold"] is not None:
|
||||
return m["error_value"] > m["error_threshold"]
|
||||
return m["corr_x"] is None
|
||||
|
||||
|
||||
def parse_time(text):
|
||||
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
|
||||
t = datetime.fromisoformat(text)
|
||||
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def iso(t):
|
||||
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Detector of the Displacement family (labeling/familles.md): sudden jumps of corr_x / corr_y (px) of each target, at
|
||||
the native cadence of the "advanced" export, and slow drift (end of file). A flipping target ("Target flipping")
|
||||
belongs to the Target issue family: episodes() returns it, detect() leaves it to target_issue.py.
|
||||
|
||||
1. Noise of the series = median of the gaps from one measurement to the next; jump = measurement K noises or more away
|
||||
from the level before (median of the BASE previous measurements); the first BASE measurements, lacking a level
|
||||
before, are compared with the level after (median of the BASE next ones), a gap there is a jump back at once.
|
||||
2. Back (an hour of measurements in a row within half the jump: BASE at 15 min, a single one at 4 h; learner
|
||||
MICHIGAN, mod. 1) within a day, or one day later back at its usual place (first day of the export) or at the day
|
||||
before's: "Sudden jump, back to normal", greyed; its cause is not established here (categories.py gives it the
|
||||
cause of the category that overlaps it, otherwise: to be checked on the images).
|
||||
3. Otherwise "Lasting shift", never greyed (maybe a real movement); less than a day of hindsight: "Jump, not back yet",
|
||||
not greyed either.
|
||||
4. Jumps following each other within GAP, all targets together, make a single episode (common mode); a single lasting
|
||||
(or too recent) target is enough not to grey the episode.
|
||||
5. Global references excluded; rejected measurements (empty corr) skipped.
|
||||
6. Common peak (NORDEND, batch 01): at the same instant, at least PEAK_MIN targets move away by at least K_PEAK noises
|
||||
from both neighbouring measurements, on the same side (the whole image shaken for one step): a jump back for each,
|
||||
even under K. It joins the episodes of point 4 (next to a lasting jump, it greys nothing).
|
||||
7. (CLARIDEN, batch 01) a jump that never comes back no longer ends the reading of the series, its new level becomes
|
||||
the level before; an episode that is not greyed ends at its last jump, not at the returns; a flipping target
|
||||
(REPEAT jumps back or more, each within a day of the previous one) makes a "back to normal" episode on its own, from
|
||||
the first to the last flip, outside the common episodes. A target that flips for at least FLIP, with at most
|
||||
MAX_FLIPPING targets at the same time, becomes "Target flipping" (Target issue, D-074); a lasting jump of at least
|
||||
LARGE of the targets, "Large shift" (everything moved), always to be checked. Learner session Target issue, mod. 1,
|
||||
S2: FLIP from 6 h to 36 h; a target flipping for a day during a nuisance of the whole scene (rain, fog, night of poor
|
||||
visibility) is not a problem target; its jumps back go to the nuisance overlapping them, otherwise stay
|
||||
Displacement.
|
||||
8. Jump under K that stays (NIEDERHORN, batch 01): a jump of at least K_LASTING noises counts if it is lasting in the
|
||||
sense of point 3 and the median of the day after moves away from that of the day before, on the jump's side, by at
|
||||
least PLATEAU times the ordinary variation of the level from one UTC day to the next (median of the gaps between
|
||||
daily levels); otherwise it is ignored (neither back nor too recent: only jumps of K noises or more are).
|
||||
"""
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent)) # data.py, next to it
|
||||
import data # noqa: E402
|
||||
|
||||
K = 20 # tuned on E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 (17 and 19
|
||||
# noises) are caught by visibility.py: do not lower K
|
||||
K_LASTING = 7 # under K, a jump that stays counts from K_LASTING noises (NIEDERHORN, batch 01: e09, corr_y −0.18 px at
|
||||
# once, 8 noises, stayed 5 days)
|
||||
PLATEAU = 8 # ... if it stays: median of the day after, on the jump's side, PLATEAU times the ordinary variation of the
|
||||
# level from one day to the next away from the day before (e09: 13; E-001, healthy targets: 6 at most)
|
||||
K_PEAK = 6 # one-step peak common to several targets (NORDEND shakes, 6 to 11 noises, batch 01)
|
||||
PEAK_MIN = 3 # targets peaking at the same step for a common peak (one or two: ordinary noise)
|
||||
BASE = 4 # measurements that make the level before (1 h at 15 min)
|
||||
GAP = timedelta(hours=1) # two jumps within an hour, all targets together: a single episode
|
||||
BACK_H = 24 # back within a day = artefact (E-002: plateaus up to 11.5 h)
|
||||
FLIP = timedelta(hours=36) # a problem target flips for several days (PR_BG: 2.5 days); less is an agitated morning
|
||||
# (E-009: C8 and C13, 5 flips in 1 h 20 during the movement) or a night of nuisance (NORDEND, JUNGFRAU:
|
||||
# 12 to 28 h within a loss of visibility), a jump back, absorbed by its host
|
||||
MAX_FLIPPING = 2 # targets flipping at the same time, at most (FLORIDA: the two stuck ones; E-002, cobweb on the
|
||||
# references: 15 at once)
|
||||
LARGE = 0.5 # share of the targets of a lasting jump for a large shift (E-004: almost all)
|
||||
REPEAT = 5 # jumps back of the same target, each within a day of the other, that make it a flipping target (PR_BG of
|
||||
# CLARIDEN, batch 01: about thirty in 2.5 days)
|
||||
|
||||
|
||||
def _jumps(t, v):
|
||||
"""A series (instants, values) -> [(start i, end i, state)], state 'back', 'lasting' or 'recent'."""
|
||||
day = timedelta(hours=BACK_H)
|
||||
noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:]))
|
||||
step = statistics.median(b - a for a, b in zip(t, t[1:])) # cadence of the series
|
||||
back_n = max(1, min(BASE, round(timedelta(hours=1) / step))) # measurements of an hour (BASE at 15 min, one at 4 h)
|
||||
# ponytail: usual place = first day of the export, wrong if it starts within an artefact
|
||||
ref = statistics.median(x for x, u in zip(v, t) if u < t[0] + day)
|
||||
lv = {} # level of each UTC day; its ordinary variation from one day to the next judges a jump under K
|
||||
for x, u in zip(v, t):
|
||||
lv.setdefault(u.date(), []).append(x)
|
||||
lv = [statistics.median(x) for _, x in sorted(lv.items())]
|
||||
ordinary = statistics.median(abs(b - a) for a, b in zip(lv, lv[1:])) if len(lv) > 2 else 0
|
||||
# the first BASE measurements, lacking a level before, face the level after: a gap there came back at once
|
||||
out = [(k, k, "back") for k in range(min(BASE, len(v) - BASE))
|
||||
if noise and abs(v[k] - statistics.median(v[k + 1:k + 1 + BASE])) >= K * noise]
|
||||
i = BASE
|
||||
while noise and i < len(v):
|
||||
base = statistics.median(v[i - BASE:i])
|
||||
if abs(v[i] - base) < K_LASTING * noise:
|
||||
i += 1
|
||||
continue
|
||||
tol = abs(v[i] - base) / 2 # back: an hour of measurements in a row within half the jump from the level before
|
||||
j = next((k for k in range(i + 1, len(v)) if all(abs(x - base) < tol for x in v[k:k + back_n])), len(v))
|
||||
before = statistics.median([x for x, u in zip(v, t) if t[i] - day <= u < t[i]] or [base])
|
||||
after = statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + day)
|
||||
if j < len(v) and t[j] - t[i] <= day:
|
||||
state = "back"
|
||||
elif abs(after - ref) < tol or abs(after - before) < tol: # the day after, the target is at its usual place,
|
||||
state, j = "back", i + 1 # or where it was the day before: the jump ended a dip
|
||||
else:
|
||||
state = "lasting" if t[-1] - t[i] >= day else "recent" # less than a day of hindsight: not known yet
|
||||
if abs(v[i] - base) < K * noise and (state != "lasting" or (after - before) * (v[i] - base) < 0
|
||||
or not ordinary or abs(after - before) < PLATEAU * ordinary):
|
||||
i += 1 # under K, only a jump that stays counts: the day after far from the day before, on its side
|
||||
continue
|
||||
out.append((i, j - 1, state))
|
||||
# level before rebuilt after the return; never back: the new level becomes the level before, the rest is
|
||||
# still read (PR_BG of CLARIDEN, batch 01: its flips after the 10:30 jump were invisible)
|
||||
i = (j if j < len(v) else i) + BASE
|
||||
return out
|
||||
|
||||
|
||||
def _peaks(t, v):
|
||||
"""Instants where the measurement moves away by at least K_PEAK noises from both neighbours, same side (one-step peak)."""
|
||||
noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:]))
|
||||
return [t[i] for i in range(1, len(v) - 1) if noise and (v[i] - v[i - 1]) * (v[i] - v[i + 1]) > 0
|
||||
and min(abs(v[i] - v[i - 1]), abs(v[i] - v[i + 1])) >= K_PEAK * noise]
|
||||
|
||||
|
||||
def episodes(measurements):
|
||||
"""{sensor: [measurement]} (data.load) -> jump episodes, sorted by start: Displacement, and Target issue for a
|
||||
flipping target."""
|
||||
found = [] # (start, end, state, sensor)
|
||||
peaks = {} # instant -> targets peaking there for one step
|
||||
for name, ms in measurements.items():
|
||||
if data.is_reference(ms):
|
||||
continue
|
||||
for axis in ("corr_x", "corr_y"):
|
||||
kept = [m for m in ms if m[axis] is not None]
|
||||
if len(kept) <= BASE:
|
||||
continue
|
||||
t, v = [m["t"] for m in kept], [m[axis] for m in kept]
|
||||
# a jump that did not come back ends with itself, not with the data (its bar hid the others)
|
||||
found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)]
|
||||
for u in _peaks(t, v):
|
||||
peaks.setdefault(u, set()).add(name)
|
||||
found += [(u, u, "back", s) for u, cs in peaks.items() if len(cs) >= PEAK_MIN for s in cs] # common peak: back
|
||||
alone = _flips(found) # a target that keeps jumping and coming back: an episode on its own, outside the common ones
|
||||
found = [f for f in found if f[2] != "back" or not any(f[3] in g["sensors"] and g["start"] <= f[0] <= g["end"]
|
||||
for g in alone)]
|
||||
groups = [] # a jump that did not come back only counts by its instant: its end is often the end of the data
|
||||
for s, e, state, name in sorted(found):
|
||||
if groups and s <= groups[-1]["until"] + GAP:
|
||||
g = groups[-1]
|
||||
else:
|
||||
g = {"start": s, "end": e, "until": s, "last": s, "sensors": {}}
|
||||
groups.append(g)
|
||||
g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s
|
||||
g["sensors"].setdefault(name, set()).add(state)
|
||||
long = [g for g in alone if g["flip"]] # flips of many targets at the same time: the compensation, not a
|
||||
for g in long: # problem target
|
||||
if sum(h["start"] <= g["end"] and h["end"] >= g["start"] for h in long) > MAX_FLIPPING:
|
||||
g["flip"] = 0
|
||||
total = sum(not data.is_reference(ms) for ms in measurements.values())
|
||||
return sorted((_episode(g, total) for g in groups + alone), key=lambda ep: ep["start"])
|
||||
|
||||
|
||||
def _flips(found):
|
||||
"""Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> a group, from the first
|
||||
to the last flip."""
|
||||
day, alone = timedelta(hours=BACK_H), []
|
||||
for name in {f[3] for f in found}:
|
||||
chain = [] # (start, end); corr_x and corr_y twice: counted by instant
|
||||
for s, e in sorted({(s, e) for s, e, state, n in found if n == name and state == "back"}) + [(None, None)]:
|
||||
if chain and s is not None and s <= max(c[1] for c in chain) + day:
|
||||
chain.append((s, e))
|
||||
continue
|
||||
if len({c[0] for c in chain}) >= REPEAT:
|
||||
alone.append({"start": chain[0][0], "end": max(c[1] for c in chain), "sensors": {name: {"back"}},
|
||||
"flip": len({c[0] for c in chain}) if max(c[1] for c in chain) - chain[0][0] >= FLIP else 0})
|
||||
chain = [(s, e)]
|
||||
return alone
|
||||
|
||||
|
||||
def _episode(g, total):
|
||||
"""A group -> episode; a single lasting (or too recent) jump is enough not to grey; a flipping target: "Target
|
||||
flipping" (Target issue, D-074); a lasting jump of LARGE targets or more: "Large shift"."""
|
||||
n = len(g["sensors"])
|
||||
lasting = sum("lasting" in st for st in g["sensors"].values())
|
||||
recent = any("recent" in st for st in g["sensors"].values())
|
||||
who = "One target" if n == 1 else f"{n} targets together (a single event)"
|
||||
if g.get("flip"):
|
||||
name = next(iter(g["sensors"]))
|
||||
return {"start": data.iso(g["start"]), "end": data.iso(g["end"]), "kind": "Target issue", "label": "Target flipping",
|
||||
"text": f"{name} jumped and came back {g['flip']} times: its tracking is not reliable, look at the images "
|
||||
"of the target.", "ignore": True, "sensors": [name], "lasting": []}
|
||||
if lasting and n >= LARGE * total:
|
||||
label = "Large shift"
|
||||
text = f"{n} targets jumped together and {lasting} of them stayed shifted: everything moved, please check it."
|
||||
elif lasting:
|
||||
label = "Lasting shift"
|
||||
some = "" if lasting == n else f", {lasting} of them stayed shifted for more than a day"
|
||||
text = (f"{who} jumped{some}" if some else f"{who} jumped and stayed shifted for more than a day") + \
|
||||
": this may be a real movement, please check it."
|
||||
elif recent:
|
||||
label = "Jump, not back yet"
|
||||
text = f"{who} jumped less than a day before the last measurement and has not come back yet: check again later."
|
||||
else:
|
||||
label = "Sudden jump, back to normal"
|
||||
text = f"{who} jumped and came back within a day: the cause is not established, look at the images."
|
||||
end = g["last"] if lasting or recent else g["end"] # not greyed: the last jump, not the returns
|
||||
return {"start": data.iso(g["start"]), "end": data.iso(end), "kind": "Displacement", "label": label,
|
||||
"text": text, "ignore": not lasting and not recent, "sensors": sorted(g["sensors"]),
|
||||
"lasting": sorted(s for s, st in g["sensors"].items() if "lasting" in st)} # stayed shifted (event sentence)
|
||||
|
||||
|
||||
def detect(measurements):
|
||||
return sorted([e for e in episodes(measurements) if e["kind"] == "Displacement"] + drift(measurements),
|
||||
key=lambda ep: ep["start"])
|
||||
|
||||
|
||||
# --- Slow drift ------------------------------------------------------------------------------------------------------
|
||||
# A target that moves little by little, day after day. Takes up the trend of the Displacements tab (D-055) on the
|
||||
# "advanced" export (corr_x, corr_y, px). Method, per target (global references excluded) and per axis:
|
||||
# 1. Level of the day: median of the measurements of each UTC day with at least HOURS measured hours out of 24, as a
|
||||
# share of the hours the station measures (learner MICHIGAN, mod. 1: at one measurement every 4 h, 5 of the 6 hours;
|
||||
# before, no day counted); at least DAYS days.
|
||||
# 2. Theil-Sen slope and Kendall tau of the daily levels; amplitude of the day: gap between its 9th and 1st deciles.
|
||||
# 3. Drift: |slope| × duration ≥ SIZE times the median daily amplitude, |tau| ≥ STEADY, and no step from one day to the
|
||||
# next of more than half the drift (a jump that lasts is not a drift: the jump rule takes it).
|
||||
# 4. One episode per target (one axis is enough), from the first measurement of the first day to the last of the last
|
||||
# day; not greyed.
|
||||
# 4b. Piecewise (learner session Displacement, mod. 1, S1): a target without a drift over the whole window may drift for
|
||||
# a few days, then settle or go back. On each run of at least DAYS kept days, drift = level of the last day minus
|
||||
# that of the first, at least PIECE daily amplitudes (median of the series), |tau| ≥ STEADY with the same sign, no
|
||||
# step from one kept day to the next of more than half the drift; pieces taken from the largest drift to the
|
||||
# smallest, without overlapping (they may share the day where the level turns): each goes from one extreme of the
|
||||
# level to the other, one episode per piece and per axis. At PIECE = 1, the thermal cycles of a few days and the
|
||||
# noise of the levels get through (JUNGFRAU, NORDEND): 3.5 keeps the rows of CLARIDEN (3.8 to 8.3 amplitudes),
|
||||
# drops JUNGFRAU (3.1 at most). The pieces do not replace step 3. A piece does not cross a hole: two consecutive kept
|
||||
# days are at most MAX_HOLE days apart (mod. 2: the gap of a level across three days without a measurement is a
|
||||
# step, not a slope; it made pieces of the whole window, CLARIDEN 09-28 → 11-01). A missing day does not cut.
|
||||
# 5. Camera drift (same mod.): at least LARGE of the targets drift on the same side on corr_x or corr_y, and at least
|
||||
# LARGE of the sensors, references included, on their raw positions x or y: only the pieces of point 4b are
|
||||
# reported, not the trend over the whole window (mod. 1, S2: the badly compensated drifting camera, NEBRASKA, is a
|
||||
# Displacement for the labels). The targets of a bridge following the season (JUNGFRAU, labelled Thermal
|
||||
# displacement day after day) pass step 3 but not PIECE; NEBRASKA both.
|
||||
HOURS = 20 # a day counts if it has at least 20 measured hours out of 24 (as D-055), as a share of the measured hours
|
||||
DAYS = 5 # minimum days for a trend (D-055)
|
||||
SIZE = 1.0 # drift ≥ SIZE daily amplitudes (D-055)
|
||||
STEADY = 0.6 # |Kendall tau| of the daily levels (D-055)
|
||||
PIECE = 3.5 # piecewise drift ≥ PIECE daily amplitudes (point 4b)
|
||||
MAX_HOLE = 2 # a piece does not go from one kept day to the next more than MAX_HOLE days apart (point 4b)
|
||||
|
||||
|
||||
def _decile(vs, q):
|
||||
vs = sorted(vs)
|
||||
return vs[min(len(vs) - 1, int(q * len(vs)))]
|
||||
|
||||
|
||||
def _days(ms, col):
|
||||
"""-> {UTC day: its measurements}, the kept days (point 1)."""
|
||||
days = {}
|
||||
for m in ms:
|
||||
if m[col] is not None:
|
||||
days.setdefault(m["t"].date(), []).append(m)
|
||||
slots = len({m["t"].hour for dm in days.values() for m in dm}) # hours the station measures (6 at a 4 h step)
|
||||
return {d: dm for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots}
|
||||
|
||||
|
||||
def _amplitude(days, col):
|
||||
return statistics.median(_decile([m[col] for m in dm], 0.9) - _decile([m[col] for m in dm], 0.1) for dm in days.values())
|
||||
|
||||
|
||||
def _drift(ms, col):
|
||||
"""-> (drift in px, days, first instant, last) or None."""
|
||||
days = _days(ms, col)
|
||||
if len(days) < DAYS:
|
||||
return None
|
||||
lv = sorted((d.toordinal(), statistics.median(m[col] for m in dm)) for d, dm in days.items())
|
||||
pairs = [(a, b) for k, a in enumerate(lv) for b in lv[k + 1:]]
|
||||
slope = statistics.median((b[1] - a[1]) / (b[0] - a[0]) for a, b in pairs)
|
||||
tau = sum((b[1] > a[1]) - (b[1] < a[1]) for a, b in pairs) / len(pairs)
|
||||
drift = slope * (lv[-1][0] - lv[0][0])
|
||||
step = max(abs(b[1] - a[1]) for a, b in zip(lv, lv[1:]))
|
||||
if abs(drift) < SIZE * _amplitude(days, col) or abs(tau) < STEADY or step > abs(drift) / 2:
|
||||
return None
|
||||
d0, d1 = min(days), max(days)
|
||||
return drift, len(days), min(m["t"] for m in days[d0]), max(m["t"] for m in days[d1])
|
||||
|
||||
|
||||
def _pieces(ms, col):
|
||||
"""-> [(drift in px, days, first instant, last)]: the drift pieces (point 4b)."""
|
||||
days = _days(ms, col)
|
||||
if len(days) < DAYS:
|
||||
return []
|
||||
ds = sorted(days)
|
||||
lv = [statistics.median(m[col] for m in days[d]) for d in ds]
|
||||
ampl = _amplitude(days, col)
|
||||
pieces = [] # (drift, i, j)
|
||||
for i in range(len(ds)):
|
||||
signs = step = 0
|
||||
for j in range(i + 1, len(ds)):
|
||||
if (ds[j] - ds[j - 1]).days > MAX_HOLE:
|
||||
break # a gap across a hole is a step (the jump rule takes it), not a slope
|
||||
signs += sum((lv[j] > lv[k]) - (lv[j] < lv[k]) for k in range(i, j)) # Kendall tau, step by step
|
||||
step = max(step, abs(lv[j] - lv[j - 1]))
|
||||
n, d = j - i + 1, lv[j] - lv[i]
|
||||
if n >= DAYS and d and abs(d) >= PIECE * ampl and step <= abs(d) / 2 \
|
||||
and signs * (1 if d > 0 else -1) >= STEADY * n * (n - 1) / 2:
|
||||
pieces.append((d, i, j))
|
||||
taken = []
|
||||
for d, i, j in sorted(pieces, key=lambda x: (-abs(x[0]), x[2] - x[1])):
|
||||
if all(j <= a or i >= b for _, a, b in taken):
|
||||
taken.append((d, i, j))
|
||||
return [(d, j - i + 1, min(m["t"] for m in days[ds[i]]), max(m["t"] for m in days[ds[j]])) for d, i, j in taken]
|
||||
|
||||
|
||||
def _common(measurements, cols):
|
||||
"""At least LARGE of the sensors drift on the same side on the same axis of cols."""
|
||||
for col in cols:
|
||||
signs = [x[0] > 0 for ms in measurements.values() if (x := _drift(ms, col))]
|
||||
if max(signs.count(True), signs.count(False)) >= LARGE * len(measurements):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def drift(measurements):
|
||||
targets = {s: ms for s, ms in measurements.items() if not data.is_reference(ms)}
|
||||
camera = _common(targets, ("corr_x", "corr_y")) and _common(measurements, ("x", "y")) # point 5: pieces only
|
||||
out = []
|
||||
for s, ms in sorted(targets.items()):
|
||||
r = [] if camera else [x for col in ("corr_x", "corr_y") if (x := _drift(ms, col))]
|
||||
r = [max(r, key=lambda x: abs(x[0]))] if r else [x for col in ("corr_x", "corr_y") for x in _pieces(ms, col)]
|
||||
out += [{"start": data.iso(a), "end": data.iso(b), "kind": "Displacement", "label": "Slow drift",
|
||||
"text": f"{s} drifts slowly and steadily: {d:+.2f} px in {n} days. Keep watching it.",
|
||||
"ignore": False, "sensors": [s]} for d, n, a, b in r]
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for ep in detect(data.load(sys.argv[1])):
|
||||
print(ep["start"], ep["end"], "ignore" if ep["ignore"] else "KEEP ", ep["label"], "|", len(ep["sensors"]),
|
||||
",".join(ep["sensors"][:6]), "|", ep["text"])
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Benchmark of the learner (D-116): the detectors against all the labeller's labels (labeling/auto/).
|
||||
|
||||
For each station of labeling/auto/ that has its folder in data/, the score per family and per instant of score.py
|
||||
(D-112), over the range of its labels: from the day of the first to the midnight after the last. Then the total, family
|
||||
by family, which says whether the detectors do better overall from one session to the next. The block of the session's
|
||||
station is its score; the other blocks, the non-regression: a family that goes back on a station is a regression, even
|
||||
if the total rises.
|
||||
Usage: python3 detectors/scripts/learner/regression.py [--stations A,B]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from datetime import timedelta
|
||||
|
||||
import score # next to it
|
||||
|
||||
from data import iso, parse_time # on the path through score
|
||||
|
||||
|
||||
def label_range(path):
|
||||
"""labeling/auto/<STATION>.jsonl -> (start, end) iso, in whole UTC days, end excluded."""
|
||||
with open(path, encoding="utf-8") as f:
|
||||
labels = [json.loads(l) for l in f if l.strip()]
|
||||
start = min(e["debut"] for e in labels)[:10]
|
||||
# ponytail: the end of the batch is not written; midnight closest to the last bound (a label bound overflows by one
|
||||
# measurement after midnight); the real end of the batch in labeling/auto/ if the gap matters
|
||||
end = iso(parse_time(max(e["fin"] for e in labels)) + timedelta(hours=12))[:10]
|
||||
return iso(parse_time(start)), iso(parse_time(end))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("--stations", help="A,B: these stations only (a shorter run)")
|
||||
a = p.parse_args()
|
||||
total = {}
|
||||
for path in sorted((score.WORK / "labeling" / "auto").glob("*.jsonl")):
|
||||
station = path.stem
|
||||
if a.stations and station not in a.stations.split(","):
|
||||
continue
|
||||
folder = score.WORK / "data" / station
|
||||
if not folder.is_dir():
|
||||
print(f"{station}: no data/{station}/, skipped\n")
|
||||
continue
|
||||
start, end = label_range(path)
|
||||
n, labels = score.family_scores(str(folder), start, end)
|
||||
score.show(f"{station}: {start} → {end}, {len(labels)} labels", n)
|
||||
print()
|
||||
for k, v in n.items():
|
||||
total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)]
|
||||
score.show("Total", total)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Score of the detectors on a range: the detectors' events against the labels of labeling/auto/<STATION>.jsonl that
|
||||
touch the range, per family (labeling/familles.md) and per instant (D-112). An instant: an hour of the range where at
|
||||
least one sensor has a measurement; it receives, on one side, the families of the labels covering it, bounds excluded
|
||||
(except for a hole, without a measurement inside), on the other, those of the detectors' events (categories.py, the
|
||||
range classified with its 3 days of history). The category of an event is its family.
|
||||
found: labelled instants that the detectors also flagged (share of the labelled ones);
|
||||
correct: flagged instants that a label also puts in the family (share of the flagged ones).
|
||||
The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error.
|
||||
Label fields (written by Diagnostic-app): debut, fin, categorie (the family), description.
|
||||
Run from the working repository, where detectors/ is mounted.
|
||||
Usage: python3 detectors/scripts/learner/score.py <station folder> --from YYYY-MM-DD --to YYYY-MM-DD
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(os.path.abspath(__file__)).parents[2] # the detectors repository
|
||||
WORK = REPO.parent # the working repository (labeling/, data/)
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
import categories # noqa: E402
|
||||
from data import iso, load, parse_time # noqa: E402
|
||||
|
||||
HISTORY = timedelta(days=3) # as the batch command (D-056)
|
||||
|
||||
|
||||
def events(measurements, start, end, folder=None):
|
||||
"""Detectors' events touching the range, classified with its days of history (as the batch command)."""
|
||||
h, b = parse_time(start) - HISTORY, parse_time(end)
|
||||
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
|
||||
return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start]
|
||||
|
||||
|
||||
def count(pairs):
|
||||
"""[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}."""
|
||||
n = {}
|
||||
for a, d in pairs:
|
||||
for k in a | d:
|
||||
x = n.setdefault(k, [0, 0, 0])
|
||||
x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d)
|
||||
return n
|
||||
|
||||
|
||||
def auto_labels(station, start, end):
|
||||
"""labeling/auto/<STATION>.jsonl -> the labeller's labels touching [start, end)."""
|
||||
path = WORK / "labeling" / "auto" / f"{station}.jsonl"
|
||||
if not path.exists():
|
||||
return []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return [e for e in map(json.loads, filter(str.strip, f)) if e["debut"] < end and e["fin"] >= start]
|
||||
|
||||
|
||||
def family_scores(folder, start, end):
|
||||
"""Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels)."""
|
||||
# ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the
|
||||
# label if this precision is missing
|
||||
measurements = load(folder)
|
||||
labels = auto_labels(os.path.basename(os.path.normpath(folder)), start, end)
|
||||
evts = events(measurements, start, end, folder)
|
||||
instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end})
|
||||
# bounds excluded: they are the healthy measurement before and after (D-111); a hole, without a measurement
|
||||
# inside, keeps them
|
||||
empty = {i for i, e in enumerate(labels) if not any(e["debut"] < t < e["fin"] for t in instants)}
|
||||
pairs = []
|
||||
for t in instants:
|
||||
a = {e.get("categorie", "no family") for i, e in enumerate(labels)
|
||||
if e["debut"] < t < e["fin"] or (i in empty and e["debut"] <= t <= e["fin"])} - {"None"}
|
||||
d = {v["category"] for v in evts if v["start"] <= t <= v["end"]}
|
||||
pairs.append((a, d))
|
||||
return count(pairs), labels
|
||||
|
||||
|
||||
def pct(a, b):
|
||||
return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—"
|
||||
|
||||
|
||||
def show(title, n):
|
||||
print(title)
|
||||
print(f" {'':<22}{'found':>20}{'correct':>20}")
|
||||
for k in sorted(n):
|
||||
labelled, flagged, both = n[k]
|
||||
print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("folder")
|
||||
p.add_argument("--from", dest="start", required=True)
|
||||
p.add_argument("--to", dest="end", required=True)
|
||||
a = p.parse_args()
|
||||
station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end))
|
||||
n, labels = family_scores(a.folder, start, end)
|
||||
if not labels:
|
||||
sys.exit(f"No label of {station} on {start} → {end} in labeling/auto/{station}.jsonl: analyse the range first.")
|
||||
show(f"{station}: {start} → {end}, {len(labels)} labels of labeling/auto/{station}.jsonl, per family and per instant", n)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""For the upgrader: the cadence of the measurements of the range, the number of targets and references, then each event
|
||||
of the range with its sensors (categories.py only gives their number). Same computation as categories.py without
|
||||
--chunk (3 days of history).
|
||||
Usage: python3 detectors/scripts/learner/sensors.py <folder> --from YYYY-MM-DD --to YYYY-MM-DD
|
||||
"""
|
||||
import argparse
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import categories # noqa: E402
|
||||
from data import iso, load # noqa: E402
|
||||
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("folder")
|
||||
p.add_argument("--from", dest="start", required=True)
|
||||
p.add_argument("--to", dest="end", required=True)
|
||||
a = p.parse_args()
|
||||
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
|
||||
start, end = day(a.start), day(a.end)
|
||||
batch = {s: k for s, ms in load(a.folder).items() if (k := [m for m in ms if start - timedelta(days=3) <= m["t"] < end])}
|
||||
ts = sorted({m["t"] for ms in batch.values() for m in ms if m["t"] >= start})
|
||||
steps = [(y - x).total_seconds() / 60 for x, y in zip(ts, ts[1:])]
|
||||
print(f"cadence: {statistics.median(steps):.0f} min (median of the steps between measurements)" if steps else "cadence: no measurement")
|
||||
refs = sum(ms[0]["pattern_type"] == "global_ref" for ms in batch.values())
|
||||
print(f"sensors: {len(batch) - refs} targets, {refs} references")
|
||||
for v in (v for v in categories.events(batch, end, a.folder) if v["end"] >= iso(start)) if batch else []:
|
||||
print(v["start"], "→", v["end"], f"{v['category']:<20}", "|", v["episodes"][0]["label"], "|",
|
||||
", ".join(sorted(v["sensors"])) or "all sensors")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Detector of the No data family (labeling/familles.md): moments without a measurement (the cadence of the "advanced"
|
||||
export is broken).
|
||||
|
||||
Cadence: median gap between two measurement instants, all sensors together. A hole: more than MISSING missing steps
|
||||
between two measurements (gap > MISSING + 1 cadences); the episode runs from the last measurement to the next one.
|
||||
End of the window read (end, when the caller gives it: the chunked run of categories.py): the same rule between the last
|
||||
measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of
|
||||
the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the
|
||||
export has nothing.
|
||||
Usage: python3 detectors/no_data.py <station folder> (prints the episodes)
|
||||
"""
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
|
||||
from data import iso, load # noqa: E402
|
||||
|
||||
MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23)
|
||||
|
||||
|
||||
def duration(s):
|
||||
h, m = divmod(round(s / 60), 60)
|
||||
return f"{h} h {m:02d} min" if h else f"{m} min"
|
||||
|
||||
|
||||
def detect(measurements, end=None):
|
||||
"""end: end of the window read (datetime, excluded), optional; without it, a final hole is not seen."""
|
||||
ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms})
|
||||
if len(ts) < 3:
|
||||
return []
|
||||
gaps = [(b - a).total_seconds() for a, b in zip(ts, ts[1:])]
|
||||
step = statistics.median(gaps)
|
||||
out = [(a, b, g, "") for a, b, g in zip(ts, ts[1:], gaps)]
|
||||
if end: # end excluded: the last expected measurement is at end − 1 step, the gap counts as between two measurements
|
||||
out.append((ts[-1], end, (end - ts[-1]).total_seconds(), ", up to the end of the period"))
|
||||
return [{"start": iso(a), "end": iso(b), "kind": "No data", "label": "No data",
|
||||
"text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).",
|
||||
"ignore": False, "sensors": []}
|
||||
for a, b, g, tail in out if g > (MISSING + 1) * step]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for e in detect(load(sys.argv[1])):
|
||||
print(e)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Detector of the Obstruction family (labeling/familles.md): an object hides one or a few targets, for an instant or
|
||||
for long. It reuses two computations shared with other families:
|
||||
- the error_value of visibility.py: Brief obstruction (something passing, one instant) and Target obscured (a target
|
||||
rejected again and again while the others stay seen, or a nuisance that only touches a few targets, no reference);
|
||||
- the day-by-day rejections of target_issue.py: a target rejected for less than a week, Target obscured.
|
||||
A sensor already in an overlapping Target issue episode (Degraded target, Target stuck) is not returned as Obstruction:
|
||||
its curve is already to be ignored, the family is Target issue (boundary "Target issue or an episode of another family"
|
||||
of labeling/familles.md; learner session Obstruction, mod. 1: S1, problem targets of FLORIDA, NORDEND, SNCF01,
|
||||
NIEDERHORN, JUNGFRAU, CLARIDEN cut into Target obscured; S3, their isolated rejections as Brief obstruction).
|
||||
Usage: python3 detectors/obstruction.py <station folder> (prints the episodes)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it
|
||||
import target_issue # noqa: E402
|
||||
import visibility # noqa: E402
|
||||
from data import load # noqa: E402
|
||||
|
||||
|
||||
def detect(measurements):
|
||||
ti = target_issue.episodes(measurements)
|
||||
issues = [e for e in ti if e["kind"] == "Target issue"]
|
||||
out = []
|
||||
for e in visibility.episodes(measurements) + ti:
|
||||
if e["kind"] != "Obstruction":
|
||||
continue
|
||||
taken = {s for h in issues if h["start"] <= e["end"] and h["end"] >= e["start"] for s in h["sensors"]}
|
||||
if (e := {**e, "sensors": [s for s in e["sensors"] if s not in taken]})["sensors"]: # minus the sensors
|
||||
out.append(e) # already in Target issue
|
||||
return sorted(out, key=lambda e: e["start"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for e in detect(load(sys.argv[1])):
|
||||
print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Detector of the Target issue family (labeling/familles.md): a persistently problematic target: rejected most of the
|
||||
time, day after day, while the others are seen (degraded target), stuck (end of file), or flipping (the jumps of
|
||||
displacement.py, "Target flipping"). A target rejected for less than a week is hidden at times (point 4): episodes()
|
||||
returns it with the Obstruction family, which detect() leaves to obstruction.py.
|
||||
|
||||
visibility.py compares each sensor with its own usual level: a target that is almost always rejected looks normal to it
|
||||
(FLORIDA, October 2026: Massif 2 and Sellette 6 rejected half the measurements or more, every day).
|
||||
Method:
|
||||
1. Calm instant for a sensor: at most CALM of the other sensors rejected (error_value > error_threshold); weather and
|
||||
fog, which touch everyone, are thus set aside.
|
||||
2. Per UTC day and per sensor, on its calm instants (at least MIN_HOURS hours of instants at the station's cadence,
|
||||
median gap between two measurement instants; otherwise the day does not count): degraded day if at least SHARE of
|
||||
them are rejected. Cadence (learner NEBRASKA, batch 01): 12 fixed instants, "3 h at 15 min", counted no day at one
|
||||
measurement every 4 h; REFCOLH22, rejected all the time since it was lost, was not seen.
|
||||
3. Episode: at least MIN_DAYS degraded days in a row (a day that does not count does not break the run), from the first
|
||||
rejected measurement of these days to the last; ignore = True: the target's curve is not reliable.
|
||||
4. Hindered all the time or at times (boundary "Target issue or Obstruction" of labeling/familles.md; learner
|
||||
NEBRASKA, batch 01): a run of at least WEEK days is a problem target, "Degraded target"; shorter, a target hidden at
|
||||
times, "Target obscured" (Obstruction family), one episode per run of rejected measurements (the hours when it is
|
||||
seen are not part of it). BREITHORN: vehicles parked in front of COLI24 and POUI22, 2 to 5 days; NEBRASKA and
|
||||
MICHIGAN: a lost reference, 25 days; JUNGFRAU: PM_G, 63.
|
||||
For the problem target, the run is bridged: at most BRIDGE non-degraded days in a row do not break it, and it then
|
||||
needs WEEK days from its first to its last degraded day, at SHARE_WEEK of rejections over all its days ("rejections
|
||||
specific to the target almost every day of the range"); the Target obscured episodes it covers no longer exist
|
||||
(learner session Obstruction, mod. 1, S1: FLORIDA Sellette 6 and Massif 2, SNCF01 F4.3 and F4.4, NORDEND CTR01,
|
||||
NIEDERHORN e01, cut into Target obscured by a day under SHARE; BREITHORN, parked vehicles, unchanged).
|
||||
The problem target judges regularity rather than the share of the day: a day is degraded from SHARE_WEEK (a quarter)
|
||||
of its calm instants rejected, and BRIDGE is one day; the target hidden at times keeps SHARE. A target rejected every
|
||||
day at the same hours, by day only or by night only, stays under half and was not seen (learner session Target
|
||||
issue, mod. 1, S1: WILDSTRUBEL R3, 04:00 to 16:00 every day; NORDEND PR04 and CTR01, NIEDERHORN PWD_IMP_B08_024_C09
|
||||
and e01, SNCF01 F4.3).
|
||||
The cause (dirt, vegetation, setting) can only be read on the images.
|
||||
Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per UTC day, the noise of its corr_x and
|
||||
corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on
|
||||
both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at
|
||||
7e-5 px and C3 at 2e-4 against 0.013 for the others.
|
||||
Usage: python3 detectors/target_issue.py <station folder> (prints the episodes)
|
||||
"""
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it
|
||||
import displacement # noqa: E402
|
||||
from data import is_reference, iso, load # noqa: E402
|
||||
|
||||
CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant
|
||||
SHARE = 0.5 # half of the calm instants rejected: "most of the time"
|
||||
SHARE_WEEK = 0.25 # a quarter of the calm instants rejected, each day of a week or more: a problem target, even when
|
||||
# rejected by day only or by night only (learner session Target issue, mod. 1, S1)
|
||||
MIN_HOURS = 3 # hours of calm instants, at the station's cadence: below that, the day does not count
|
||||
MIN_DAYS = 2 # a single day is too often an accident (NORDEND: isolated days on 16 sensors)
|
||||
BRIDGE = 1 # non-degraded days that do not break a run of degraded days (4); at 2, the parked vehicles of BREITHORN
|
||||
# (two days, then nothing for two days, then one) would make a week at a quarter
|
||||
WEEK = 7 # degraded days in a row for a problem target; below, a target hidden at times
|
||||
STUCK = 0.05 # noise of a stuck target compared with the others (E-011: 0.006; NORDEND, E-001: 0.7 at the lowest)
|
||||
|
||||
|
||||
def is_rejected(m):
|
||||
return m["error_value"] is not None and m["error_threshold"] is not None and m["error_value"] > m["error_threshold"]
|
||||
|
||||
|
||||
def cadence(measurements):
|
||||
"""Median gap between two measurement instants, all sensors together, in hours (as no_data)."""
|
||||
ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms})
|
||||
return statistics.median((b - a).total_seconds() for a, b in zip(ts, ts[1:])) / 3600 if len(ts) > 1 else 1
|
||||
|
||||
|
||||
def episodes(measurements):
|
||||
"""Targets rejected day after day: Target issue (a week or more) and Obstruction (less), then the stuck ones."""
|
||||
min_instants = MIN_HOURS / cadence(measurements)
|
||||
by_t = {}
|
||||
for s, ms in measurements.items():
|
||||
for m in ms:
|
||||
by_t.setdefault(m["t"], {})[s] = is_rejected(m)
|
||||
days = {} # sensor -> day -> [rejected, calm, first measurement, last]
|
||||
for t in sorted(by_t):
|
||||
d = by_t[t]
|
||||
for s, r in d.items():
|
||||
others = [v for k, v in d.items() if k != s]
|
||||
c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t])
|
||||
c[3] = t
|
||||
if others and sum(others) / len(others) <= CALM:
|
||||
c[0] += r
|
||||
c[1] += 1
|
||||
out = []
|
||||
for s, ds in sorted(days.items()):
|
||||
covered = [] # bridged runs of at least WEEK days, at SHARE_WEEK of rejections: a problem target
|
||||
for run in _runs(ds, min_instants, BRIDGE, SHARE_WEEK):
|
||||
share = sum(c[0] for c in run) / sum(c[1] for c in run)
|
||||
if len(run) < WEEK or share < SHARE_WEEK:
|
||||
continue
|
||||
n = sum(c[0] / c[1] >= SHARE_WEEK for c in run)
|
||||
r = _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])
|
||||
covered.append((run[0][2], run[-1][3]))
|
||||
out.append({"start": iso(r[0][0]), "end": iso(r[-1][1]), "kind": "Target issue", "label": "Degraded target",
|
||||
"text": f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were "
|
||||
f"seen, on {n} of {len(run)} days.", "ignore": True, "sensors": [s]})
|
||||
for run in _runs(ds, min_instants, 0, SHARE): # at times: one episode per run of rejected measurements
|
||||
if len(run) < MIN_DAYS or any(a <= run[0][2] <= b for a, b in covered):
|
||||
continue
|
||||
share = round(100 * sum(c[0] for c in run) / sum(c[1] for c in run))
|
||||
text = f"{s} was rejected in {share} % of the measurements when the other targets were seen, {len(run)} days in a row."
|
||||
out += [{"start": iso(a), "end": iso(b), "kind": "Obstruction", "label": "Target obscured", "text": text,
|
||||
"ignore": True, "sensors": [s]}
|
||||
for a, b in _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])]
|
||||
return sorted(out + stuck(measurements, min_instants), key=lambda e: e["start"])
|
||||
|
||||
|
||||
def _runs(ds, min_instants, bridge, share):
|
||||
"""{day: [rejected, calm, first, last]} -> runs of degraded days; at most `bridge` non-degraded days in a row between
|
||||
two degraded days do not break the run (they belong to it); a day that does not count does not break it."""
|
||||
runs, cur, gap = [], [], []
|
||||
for j in sorted(ds):
|
||||
rej, calm = ds[j][:2]
|
||||
if calm < min_instants:
|
||||
continue
|
||||
if rej / calm >= share:
|
||||
cur += gap + [ds[j]]
|
||||
gap = []
|
||||
elif cur:
|
||||
gap.append(ds[j])
|
||||
if len(gap) > bridge:
|
||||
runs.append(cur)
|
||||
cur, gap = [], []
|
||||
return runs + ([cur] if cur else [])
|
||||
|
||||
|
||||
def _rejections(ms):
|
||||
"""Measurements of a sensor -> [(start, end)] of its runs of rejected measurements."""
|
||||
out, cur = [], []
|
||||
for m in ms + [None]:
|
||||
if m and is_rejected(m):
|
||||
cur.append(m["t"])
|
||||
elif cur:
|
||||
out.append((cur[0], cur[-1]))
|
||||
cur = []
|
||||
return out
|
||||
|
||||
|
||||
def _noise(v):
|
||||
return statistics.median(abs(y - x) for x, y in zip(v, v[1:])) if len(v) > 2 else None
|
||||
|
||||
|
||||
def stuck(measurements, min_instants):
|
||||
"""Targets whose corr_x and corr_y barely move, day after day, while the others move."""
|
||||
days = {} # day -> target -> (x noise, y noise, first measurement, last)
|
||||
for s, ms in measurements.items():
|
||||
if is_reference(ms):
|
||||
continue
|
||||
by_day = {}
|
||||
for m in ms:
|
||||
by_day.setdefault(m["t"].date(), []).append(m)
|
||||
for d, dm in by_day.items():
|
||||
b = [_noise([m[c] for m in dm if m[c] is not None]) for c in ("corr_x", "corr_y")]
|
||||
if None not in b and len(dm) >= min_instants:
|
||||
days.setdefault(d, {})[s] = (*b, dm[0]["t"], dm[-1]["t"])
|
||||
runs = {} # target -> [[stuck days in a row]]
|
||||
for d in sorted(days):
|
||||
cs = days[d]
|
||||
for s, (bx, by, a, b) in cs.items():
|
||||
others = [v for k, (x, y, *_) in cs.items() if k != s for v in (x, y)]
|
||||
if len(others) < 4:
|
||||
continue
|
||||
ref = statistics.median(others)
|
||||
ru = runs.setdefault(s, [[]])
|
||||
if ref and max(bx, by) <= STUCK * ref:
|
||||
ru[-1].append((a, b))
|
||||
elif ru[-1]:
|
||||
ru.append([])
|
||||
return [{"start": iso(k[0][0]), "end": iso(k[-1][1]), "kind": "Target issue", "label": "Target stuck",
|
||||
"text": f"{s} barely moved while the other targets did ({len(k)} day{'s' if len(k) > 1 else ''}): it may be "
|
||||
"set on a reference prism.", "ignore": True, "sensors": [s]}
|
||||
for s, ru in sorted(runs.items()) for k in ru if k]
|
||||
|
||||
|
||||
def detect(measurements):
|
||||
return sorted((e for e in episodes(measurements) + displacement.episodes(measurements) if e["kind"] == "Target issue"),
|
||||
key=lambda e: e["start"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for e in detect(load(sys.argv[1])):
|
||||
print(e)
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Detector of the Thermal displacement family (labeling/familles.md): the daily wave of the corrected displacements
|
||||
(expansion by day, contraction in the evening).
|
||||
|
||||
Method, per target (global references excluded) and per axis (corr_x, corr_y):
|
||||
1. Each UTC day of at least MIN_HOURS hours: mean of each hour minus the median of the day (removes jumps and drift).
|
||||
2. Profile = median, hour by hour, over the days; "repeatability" = median, over the days, of the correlation between
|
||||
the day and the profile of the other days. Axis concerned if repeatability ≥ REPRO. An incomplete day (at least
|
||||
PARTIAL hours, fewer than MIN_HOURS) does not enter the profile, but its likeness is measured on the hours it has: a
|
||||
hole of half a day no longer takes its episode away (learner session Thermal displacement, mod. 1, S4; at
|
||||
MIN_HOURS = 12 or 16, the incomplete days distorted the profile, NIEDERHORN went back).
|
||||
3. Cycle of the station if at least SHARE of the targets have an axis concerned, **and** (second method, asked by the
|
||||
user) if a 24 h sinusoid, a + b·sin(2πt/24 h) + c·cos(2πt/24 h) by least squares, explains at least R2_MIN of the
|
||||
curve of an axis for at least SHARE of the targets.
|
||||
4. Warm hours: night level = median of the NIGHT flattest consecutive hours of the profile; an hour is warm if, as a
|
||||
median over the axes concerned, the profile moves away from it by at least WARM times its amplitude.
|
||||
5. Two episodes per complete day that looks like the others (median correlation ≥ DAY_MATCH over the axes concerned):
|
||||
"Thermal expansion" from the start to the hour of the largest gap from the night level (the peak), then "Thermal
|
||||
contraction" until the return. The warm hours say there is a cycle (at least MIN_WARM_HOURS of them); its bounds
|
||||
are read lower, at EDGE times the amplitude: the start, going back from the peak while the gap stays ≥ EDGE (at most
|
||||
to the calmest hour of the profile); the return, going down from the peak until it goes back under EDGE, after
|
||||
midnight if needed, at most to the next day's start: a slow return at night chains the cycles (learner session
|
||||
Thermal displacement, mod. 1, S1). The direction (a rising or falling curve) depends on the target's orientation:
|
||||
it is not looked at, only the gap from the night level.
|
||||
Reading: the cycle is almost everywhere in the raw positions (common mode) and the compensation removes it, except at
|
||||
PS des Coudres where it stays in corr, with the same shape as in the raw positions.
|
||||
Camera thermal effect (D-074): without a cycle in corr, the same method on the raw positions x, y gives "Camera thermal
|
||||
expansion" and "Camera thermal contraction": the camera moves with the heat of its mount, the compensation removes it;
|
||||
same categories, own sentence. On the raw positions the second method is not asked (WILDSTRUBEL, batch 01: the camera
|
||||
rose by a 3 px block at sunrise, held 3 h and dropped in one step around 09:30, six days in a row, which the sinusoid
|
||||
does not explain); in exchange, a single warm hour is not a cycle (MIN_WARM_HOURS: NIEDERHORN, E-005, hourly, would
|
||||
otherwise have its noon hour as a camera contraction). The camera episode covers all sensors, references included: the
|
||||
whole image moves.
|
||||
Common slide, without a cycle (learner MICHIGAN, mod. 1): lacking a cycle in the corrected curves as in the raw
|
||||
positions, an instant is a camera slide if the raw positions (x or y) of at least SHARE of the sensors, references
|
||||
included, **and** the corrected curves (corr_x or corr_y) of at least SHARE of the targets move away on the same side by
|
||||
at least SLIDE_K noises (median of the steps of the series) from their median of the day before; slides less than a
|
||||
day apart make one episode, "Camera thermal movement", on all sensors, host of the jumps back (categories.HOSTS). The
|
||||
condition on the raw positions leaves to the jumps a flip of the computed pose alone (raw positions still).
|
||||
The camera moves with the cold (learner session Thermal displacement, mod. 1, S3): lacking a cycle, also a "Camera
|
||||
thermal movement" where the camera pose (tilt, rotation or scale) moves away from its median of the day before by at
|
||||
least COLD noises, while the station temperature (detect declares it, D-124) falls under its own by at least COLD
|
||||
noises the same hour; noise: the median of the variations over an hour (over one step, at one measurement every 4 h),
|
||||
so that the threshold does not depend on the cadence. The targets may move in opposite directions or stay almost still
|
||||
(the compensation absorbs the movement): the slide rule does not see it. Consecutive pose instants make one episode.
|
||||
Without a pose or without a temperature, nothing. The heat is not taken: in fine weather, it is the day's cycle.
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
|
||||
from data import iso, load # noqa: E402
|
||||
|
||||
MIN_HOURS = 20 # a UTC day counts in the profile if it has at least 20 measured hours
|
||||
PARTIAL = 12 # an incomplete day of at least 12 hours gets its episode if it looks like the profile
|
||||
REPRO = 0.6 # an axis has a cycle if each day looks like the others (pure noise ≈ 0)
|
||||
SHARE = 0.5 # share of the targets concerned; 2026-10_NORDEND is just under (9/19)
|
||||
WARM = 0.3 # gap from the night level, as a share of the profile's amplitude
|
||||
DAY_MATCH = 0.5 # likeness of a day to the others to give it its episode
|
||||
NIGHT = 8 # flattest consecutive hours of the profile = night level
|
||||
R2_MIN = 0.3 # draft of 2026-10-24: PS des Coudres 14 axes out of 22 above, CLARIDEN 0 out of 48
|
||||
MIN_WARM_HOURS = 2 # WILDSTRUBEL, batch 01; NIEDERHORN (E-005) has only one warm hour (noon)
|
||||
EDGE = 0.1 # bounds of the wave (start, return), as a share of the amplitude; WARM says whether there is a cycle
|
||||
SLIDE_K = 3 # common slide, in noises (median of the steps) away from the median of the day before
|
||||
COLD = 4 # the camera moves with the cold, in one-hour noises away from the day before (pose, temperature)
|
||||
W = 2 * math.pi / 86400 # angular frequency of a 24 h cycle, in rad/s
|
||||
|
||||
|
||||
def _corr(a, b):
|
||||
ma, mb = statistics.mean(a), statistics.mean(b)
|
||||
num = sum((x - ma) * (y - mb) for x, y in zip(a, b))
|
||||
den = (sum((x - ma) ** 2 for x in a) * sum((y - mb) ** 2 for y in b)) ** 0.5
|
||||
return num / den if den else 0.0
|
||||
|
||||
|
||||
def _axis(ms, col):
|
||||
"""-> (24 h profile, {day: correlation with the profile of the other days}) or None."""
|
||||
days = {}
|
||||
for m in ms:
|
||||
if m[col] is not None:
|
||||
days.setdefault(m["t"].date(), {}).setdefault(m["t"].hour, []).append(m[col])
|
||||
full, every = {}, {}
|
||||
for d, hs in days.items():
|
||||
if len(hs) >= PARTIAL:
|
||||
med = statistics.median(v for l in hs.values() for v in l)
|
||||
every[d] = {h: statistics.mean(l) - med for h, l in hs.items()}
|
||||
if len(hs) >= MIN_HOURS:
|
||||
full[d] = every[d]
|
||||
if len(full) < 2:
|
||||
return None
|
||||
profile = [statistics.median(q[h] for q in full.values() if h in q) for h in range(24) if any(h in q for q in full.values())]
|
||||
if len(profile) < 24:
|
||||
return None
|
||||
likeness = {}
|
||||
for d, p in every.items():
|
||||
others = [q for e, q in full.items() if e != d]
|
||||
hs = [h for h in p if all(h in q for q in others)]
|
||||
if len(hs) >= 3:
|
||||
likeness[d] = _corr([p[h] for h in hs], [statistics.median(q[h] for q in others) for h in hs])
|
||||
return profile, likeness
|
||||
|
||||
|
||||
def sine_r2(ms, col):
|
||||
"""Share of the curve explained by a 24 h sinusoid (least squares, 3 unknowns), or None if too short."""
|
||||
pts = [(m["t"].timestamp(), m[col]) for m in ms if m[col] is not None]
|
||||
if len(pts) <= 48:
|
||||
return None
|
||||
rows = [(1.0, math.sin(W * t), math.cos(W * t)) for t, _ in pts]
|
||||
A = [[sum(r[i] * r[j] for r in rows) for j in range(3)] for i in range(3)]
|
||||
B = [sum(r[i] * y for r, (_, y) in zip(rows, pts)) for i in range(3)]
|
||||
for i in range(3): # Gaussian elimination
|
||||
for k in range(i + 1, 3):
|
||||
f = A[k][i] / A[i][i]
|
||||
A[k] = [x - f * y for x, y in zip(A[k], A[i])]
|
||||
B[k] -= f * B[i]
|
||||
x = [0.0] * 3
|
||||
for i in (2, 1, 0):
|
||||
x[i] = (B[i] - sum(A[i][j] * x[j] for j in range(i + 1, 3))) / A[i][i]
|
||||
m = statistics.mean(y for _, y in pts)
|
||||
sst = sum((y - m) ** 2 for _, y in pts)
|
||||
sse = sum((y - sum(c * v for c, v in zip(x, r))) ** 2 for r, (_, y) in zip(rows, pts))
|
||||
return 1 - sse / sst if sst else None
|
||||
|
||||
|
||||
def sine(measurements, cols=("corr_x", "corr_y")):
|
||||
"""-> (targets with an axis following a 24 h sinusoid, targets evaluated)."""
|
||||
yes, evaluated = set(), set()
|
||||
for s, ms in measurements.items():
|
||||
if ms[0]["pattern_type"] == "global_ref":
|
||||
continue
|
||||
for col in cols:
|
||||
r2 = sine_r2(ms, col)
|
||||
if r2 is not None:
|
||||
evaluated.add(s)
|
||||
if r2 >= R2_MIN:
|
||||
yes.add(s)
|
||||
return yes, evaluated
|
||||
|
||||
|
||||
def cycle(measurements, cols=("corr_x", "corr_y")):
|
||||
"""-> (axes concerned [(sensor, col, profile, likeness)], number of targets evaluated)."""
|
||||
concerned, targets = [], set()
|
||||
for s, ms in measurements.items():
|
||||
if ms[0]["pattern_type"] == "global_ref":
|
||||
continue
|
||||
for col in cols:
|
||||
r = _axis(ms, col)
|
||||
if r is None:
|
||||
continue
|
||||
targets.add(s)
|
||||
if statistics.median(r[1].values()) >= REPRO:
|
||||
concerned.append((s, col) + r)
|
||||
return concerned, len(targets)
|
||||
|
||||
|
||||
def gaps(concerned):
|
||||
"""Median gap (over the axes concerned) from the night level, hour by hour, as a share of the profile's amplitude."""
|
||||
def score(p, h):
|
||||
start = min(range(24), key=lambda i: max(p[(i + k) % 24] for k in range(NIGHT)) - min(p[(i + k) % 24] for k in range(NIGHT)))
|
||||
night = statistics.median(p[(start + k) % 24] for k in range(NIGHT))
|
||||
return abs(p[h] - night) / (max(p) - min(p))
|
||||
return [statistics.median(score(c[2], h) for c in concerned) for h in range(24)]
|
||||
|
||||
|
||||
def warm_hours(concerned):
|
||||
return [h for h, e in enumerate(gaps(concerned)) if e >= WARM]
|
||||
|
||||
|
||||
def detect(measurements, temperature=(), camera=()):
|
||||
"""Cycle of the corrected curves (the structure); otherwise, that of the raw positions (the camera moving with the
|
||||
expansion of its mount, compensated: D-074); otherwise, the camera slides (with the cold)."""
|
||||
return (_episodes(measurements, ("corr_x", "corr_y"), "Thermal", False) or
|
||||
_episodes(measurements, ("x", "y"), "Camera thermal", True) or slides(measurements, temperature, camera))
|
||||
|
||||
|
||||
def _signs(ms, col, hourly=False, k=None):
|
||||
"""{instant: +1 or -1} where the measurement moves away by at least k (default SLIDE_K) noises from the median of the
|
||||
day before; noise: the median of the steps from one measurement to the next or, hourly, of the variations over an
|
||||
hour (one step if the cadence is slow)."""
|
||||
pts = [(m["t"], m[col]) for m in ms if m[col] is not None]
|
||||
if len(pts) < 3:
|
||||
return {}
|
||||
step = statistics.median((b - a).total_seconds() for (a, _), (b, _) in zip(pts, pts[1:])) if hourly else 0
|
||||
n = max(1, round(3600 / step)) if step else 1
|
||||
noise = statistics.median(abs(pts[i + n][1] - pts[i][1]) for i in range(len(pts) - n)) if len(pts) > n else 0
|
||||
out, j = {}, 0
|
||||
for i, (t, v) in enumerate(pts):
|
||||
while pts[j][0] < t - timedelta(days=1):
|
||||
j += 1
|
||||
before = [w for _, w in pts[j:i]]
|
||||
if noise and len(before) >= 3 and abs(v - statistics.median(before)) >= (k or SLIDE_K) * noise:
|
||||
out[t] = 1 if v > statistics.median(before) else -1
|
||||
return out
|
||||
|
||||
|
||||
def _common(series):
|
||||
"""[{instant: sign}] of a group of sensors -> instants where at least SHARE of them move away on the same side."""
|
||||
n, count = len(series), {}
|
||||
for sg in series:
|
||||
for t, x in sg.items():
|
||||
count[(t, x)] = count.get((t, x), 0) + 1
|
||||
return {t for (t, x), c in count.items() if n and c >= SHARE * n}
|
||||
|
||||
|
||||
def _cold(measurements, temperature, camera):
|
||||
"""[(start, end)] of the runs of instants where the camera pose (tilt, rotation or scale) moves away from its day
|
||||
before while the temperature falls under its own, by at least COLD one-hour noises; empty without one of them."""
|
||||
ts = [m["t"] for ms in measurements.values() for m in ms]
|
||||
hour = lambda t: t.replace(minute=0, second=0, microsecond=0) # noqa: E731
|
||||
cold = {hour(t) for t, x in _signs([{"t": t, "T": v} for t, v in temperature], "T", True, COLD).items() if x < 0}
|
||||
pose = [p for p in camera if min(ts) <= p["t"] <= max(ts)] if ts else []
|
||||
seen = {t for c in ("tilt", "rotation", "scale") for t in _signs(pose, c, True, COLD) if hour(t) in cold}
|
||||
episodes = [] # consecutive pose instants make one episode
|
||||
for i, p in enumerate(pose):
|
||||
if p["t"] in seen:
|
||||
if episodes and episodes[-1][2] == i - 1:
|
||||
episodes[-1][1:] = [p["t"], i]
|
||||
else:
|
||||
episodes.append([p["t"], p["t"], i])
|
||||
return [(a, b) for a, b, _ in episodes]
|
||||
|
||||
|
||||
def slides(measurements, temperature=(), camera=()):
|
||||
"""Common camera slide, without a 24 h cycle: the raw positions of most sensors, references included, and the
|
||||
corrected curves of most targets move away together, on the same side, from their day before; or the camera pose
|
||||
moves away from it with the cold (_cold)."""
|
||||
targets = [ms for ms in measurements.values() if ms[0]["pattern_type"] != "global_ref"]
|
||||
raw = set().union(*(_common([_signs(ms, c) for ms in measurements.values()]) for c in ("x", "y")))
|
||||
corr = set().union(*(_common([_signs(ms, c) for ms in targets]) for c in ("corr_x", "corr_y")))
|
||||
episodes = []
|
||||
for t in sorted(raw & corr):
|
||||
if episodes and t - episodes[-1][1] <= timedelta(days=1): # less than a day between two: the same slide
|
||||
episodes[-1][1] = t
|
||||
else:
|
||||
episodes.append([t, t])
|
||||
return [{"start": iso(a), "end": iso(b), "kind": "Thermal displacement", "label": "Camera thermal movement",
|
||||
"text": "The camera moved with a change of temperature and all the curves moved with it; this is not a "
|
||||
"movement of the structure, nothing to do.", "ignore": False, "sensors": sorted(measurements)}
|
||||
for a, b in episodes + _cold(measurements, temperature, camera)]
|
||||
|
||||
|
||||
def _episodes(measurements, cols, name, camera):
|
||||
concerned, n = cycle(measurements, cols)
|
||||
sensors = sorted({c[0] for c in concerned})
|
||||
if not n or len(sensors) < SHARE * n:
|
||||
return []
|
||||
if not camera: # second method, for the structure only: the sinusoid must see it too
|
||||
yes, evaluated = sine(measurements, cols)
|
||||
if not evaluated or len(yes) < SHARE * len(evaluated):
|
||||
return []
|
||||
warm = warm_hours(concerned)
|
||||
if len(warm) < MIN_WARM_HOURS:
|
||||
return []
|
||||
e = gaps(concerned)
|
||||
peak = max(warm, key=lambda h: e[h]) # the largest gap from the night level: end of the expansion
|
||||
rest = min(range(24), key=lambda h: e[h])
|
||||
a = peak # start: going back from the peak while the profile stays above EDGE, up to the calmest hour
|
||||
while (a - 1) % 24 != rest and e[(a - 1) % 24] >= EDGE:
|
||||
a -= 1
|
||||
b = peak + 1 # return: until the profile goes back under EDGE, at most to the next start
|
||||
while b < a + 24 and e[b % 24] >= EDGE:
|
||||
b += 1
|
||||
last = max(m["t"] for ms in measurements.values() for m in ms)
|
||||
days = sorted({d for c in concerned for d in c[3]})
|
||||
if camera: # the whole image moves, references included
|
||||
sensors = sorted(measurements)
|
||||
episodes = []
|
||||
for d in days: # ponytail: days of at least PARTIAL hours, the export's ends (22:00 → midnight) get no episode
|
||||
if statistics.median(c[3][d] for c in concerned if d in c[3]) < DAY_MATCH:
|
||||
continue
|
||||
midnight = datetime(d.year, d.month, d.day, tzinfo=timezone.utc)
|
||||
for label, u, v in ((f"{name} expansion", a, peak), (f"{name} contraction", peak, b)):
|
||||
start, end = midnight + timedelta(hours=u), min(midnight + timedelta(hours=v), last)
|
||||
if end > start:
|
||||
episodes.append({"start": iso(start), "end": iso(end),
|
||||
"kind": "Thermal displacement", "label": label, "text": "", "ignore": False, "sensors": sensors})
|
||||
return episodes
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
measurements = load(sys.argv[1])
|
||||
for e in detect(measurements):
|
||||
print(e["start"], "→", e["end"], e["label"], len(e["sensors"]), "targets")
|
||||
for cols in (("corr_x", "corr_y"), ("x", "y")):
|
||||
concerned, n = cycle(measurements, cols)
|
||||
k = len({c[0] for c in concerned})
|
||||
amp = statistics.median(max(c[2]) - min(c[2]) for c in concerned) if concerned else 0
|
||||
yes, evaluated = sine(measurements, cols)
|
||||
print(f"{cols[0][:-2] or 'raw'}: targets {k}/{n}, median amplitude {amp:.3f} px, warm hours (UTC) "
|
||||
f"{warm_hours(concerned) if concerned else '-'}; sinusoid: targets {len(yes)}/{len(evaluated)}")
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Detector of the Visibility family (labeling/familles.md): the camera sees less well, the image is faint or blurred
|
||||
(error_value), or it trembles (heat shimmer, end of file). The same error_value computation also finds objects in front
|
||||
of a few targets (Brief obstruction, Target obscured): episodes() returns them with the Obstruction family, which
|
||||
detect() leaves to obstruction.py.
|
||||
|
||||
Method:
|
||||
1. "High" measurement: error_value > median + K_MAD MAD of the sensor over the export, and > FACTOR × median, or
|
||||
rejected (> error_threshold) and > REJECT_FACTOR × median: a sensor at a usual level of 0.6 never doubles, and
|
||||
FLORIDA (batch 01) showed its fogs as Target obscured.
|
||||
2. Brief obstruction: an instant where at least BRIEF_MIN sensors are high, SALIENCE times more than at the neighbouring
|
||||
instants (train, vehicle), with no reference (global_ref) among them: a one-step peak touching the references is not
|
||||
an obstacle in front of targets (peak of the whole image, NORDEND on 01.09 11:15, batch 01); it is not shown. Unless
|
||||
at least BRIEF_MIN targets are rejected at this instant and neither at the instant before nor after: something
|
||||
passing in front of them, a reference touched as well (learner session Displacement, mod. 1, S5: trains of
|
||||
NIEDERHORN, 12 and 13 targets rejected).
|
||||
Also, something passing in front of one or two targets, or a single reference: at an instant, fewer than BRIEF_MIN
|
||||
sensors rejected at more than FACTOR times their median (without K_MAD: by day, heat shimmer inflates the MAD), at
|
||||
most one reference, none of them rejected at the instant before or after, and fewer than BRIEF_MIN high sensors
|
||||
(learner session Obstruction, mod. 1, S3: vehicles, passers-by, insects in front of one or two targets of
|
||||
WILDSTRUBEL, NIEDERHORN, LENZSPITZE; a reference hidden by a truck at JUNGFRAU, BREITHORN).
|
||||
3. Poor visibility: runs of at least RUN_MIN high measurements per sensor, joined across sensors (at least 2), reduced
|
||||
to the instants where at least FRAC of the episode's peak is high.
|
||||
4. Target obscured: a target (not a global_ref) rejected (> error_threshold) and above K_TARGET MAD, at least TARGET_MIN
|
||||
times less than TARGET_GAP apart, mostly outside the episodes of 2 and 3 (leaves, snow, glare on a target). The gap
|
||||
is at least two measurement steps, and two rejections are enough when TARGET_MIN steps do not fit in TARGET_GAP
|
||||
(learner session Obstruction, mod. 1, S6: at one measurement every 4 h, vehicles parked at night in front of COLI22,
|
||||
COLI24, POUI22 of BREITHORN make only two or three rejections). All episodes: ignore = True.
|
||||
5. Level of a Poor visibility (validated 2026-10-24), measurement by measurement: "none" (nothing is seen) when at least
|
||||
HALF of the usually seen targets (under their threshold at least half the time) are rejected, "reduced" (average
|
||||
data) otherwise; one episode per period of the same level, with the level field.
|
||||
6. Bounds of a Poor visibility (WILDSTRUBEL, batch 01): from the first to the last measurement where a target is
|
||||
rejected (> error_threshold), and no episode without a rejected target. On a road in full sun, the daytime
|
||||
error_value (×1.5 to ×3, heat shimmer) made reduced visibility last for hours without any rejection; references do
|
||||
not count: R3 is above its threshold almost all day.
|
||||
7. Poor visibility or object (boundary "Visibility or Obstruction" of labeling/familles.md; BREITHORN, batch 01): a
|
||||
nuisance touching at most FEW targets, and no reference, is not diffuse: it is an object in front of them (a parked
|
||||
vehicle), a "Target obscured" episode of its whole duration, without a level.
|
||||
8. Diffuse nuisance, with or without rejections (learner session Visibility, mod. 1, S2): at each instant, the median,
|
||||
over the sensors, of the error_value in MAD above each one's median; at least RUN_MIN instants in a row at K_DIFFUSE
|
||||
or more make a Poor visibility "reduced". A diffuse nuisance entirely within heat shimmer is the heat shimmer: no
|
||||
episode (otherwise it absorbed the jump back of the day before, WILDSTRUBEL on 04.10). Finds the drops of
|
||||
visibility of the whole scene without rejection at JUNGFRAU, FLORIDA, NORDEND; at 2, nights labelled thermal at
|
||||
JUNGFRAU; at 3, half the gain.
|
||||
|
||||
A simpler version (share of targets rejected above their threshold, without comparing each sensor with its own level)
|
||||
was tried on 2026-10-24 and dropped: worse on five data sets out of six (git history of Support-Agent,
|
||||
detectors/archive/visibilite_simple.py).
|
||||
"""
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
|
||||
from data import iso, load # noqa: E402
|
||||
|
||||
K_MAD = 10 # tuned on E-001 (fog: 11 to 57 MAD) and E-005 (rain, trains: 10 to 70 MAD)
|
||||
FACTOR = 2 # drops sensors with a near-zero MAD (E-004: PWD_REF10 ×1.6 at 14 MAD, E-005)
|
||||
RUN_MIN = 3 # high measurements in a row for poor visibility (E-001, E-005 rain)
|
||||
BRIEF_MIN = 3 # tuned on E-005 (trains: 5 to 17 sensors; the one with 2 targets is missed)
|
||||
K_TARGET = 5 # tuned on E-005 (leaves: 6 to 25 MAD, rejected)
|
||||
TARGET_MIN = 3 # repeated rejections of a target
|
||||
FRAC = 0.15 # cuts the tails of PR09 + CTR01 (NORDEND October) without cutting the cobweb (E-002)
|
||||
GAP_STEPS = 2.5 # steps tolerated between two high measurements of the same run (a missing step, clock to the second)
|
||||
BREAK = timedelta(hours=2) # lull separating two episodes (NORDEND October, night of 13 to 14.10)
|
||||
SALIENCE = 3 # a train stands out at least 3 times above the neighbouring instants (E-005: 5 to 17 against 0 to 2;
|
||||
# the cobweb of E-002 varies from 2 to 9 from one step to the next)
|
||||
TARGET_GAP = timedelta(hours=6) # tuned on E-005 (leaves: 7 rejections in 12 h, up to 5 h apart)
|
||||
REJECT_FACTOR = 1.2 # a rejection at 10 MAD also counts (FLORIDA, batch 01: levels of 0.6, ×2 out of reach); drops C24
|
||||
# of E-009 (level 0.68 just under the threshold 0.7, rejected at ×1.04)
|
||||
HALF = 0.5 # "nothing is seen"; tuned on NORDEND (09.10 23:00: 68 % rejected) and E-005 (rain: 24 to 33 %)
|
||||
K_DIFFUSE = 2.5 # diffuse nuisance (8); learner session Visibility, mod. 1, S2
|
||||
FEW = 3 # Poor visibility on at most 3 targets, no reference: an object, Target obscured (7); BREITHORN,
|
||||
# batch 01 (vehicles parked in front of 2 or 3 targets out of 12)
|
||||
|
||||
|
||||
def _step(measurements):
|
||||
ts = sorted({m["t"] for v in measurements.values() for m in v})
|
||||
return statistics.median(b - a for a, b in zip(ts, ts[1:]))
|
||||
|
||||
|
||||
def _group(points, gap):
|
||||
"""[(t, sensor)] sorted by t -> groups [(start, end, {sensors}, [t])] whose points are ≤ gap from each other."""
|
||||
groups = []
|
||||
for t, s in points:
|
||||
if groups and t - groups[-1][1] <= gap:
|
||||
g = groups[-1]
|
||||
groups[-1] = (g[0], t, g[2] | {s}, g[3] + [t])
|
||||
else:
|
||||
groups.append((t, t, {s}, [t]))
|
||||
return groups
|
||||
|
||||
|
||||
def _group_intervals(intervals, gap):
|
||||
"""[(start, end, sensor)] sorted -> [(start, end, {sensors})] of the intervals that overlap (within gap)."""
|
||||
out = []
|
||||
for a, b, s in intervals:
|
||||
if out and a - out[-1][1] <= gap:
|
||||
out[-1] = (out[-1][0], max(out[-1][1], b), out[-1][2] | {s})
|
||||
else:
|
||||
out.append((a, b, {s}))
|
||||
return out
|
||||
|
||||
|
||||
def _episode(start, end, label, text, sensors, level=None):
|
||||
e = {"start": iso(start), "end": iso(end), "kind": "Visibility" if label == "Poor visibility" else "Obstruction",
|
||||
"label": label, "text": text,
|
||||
"ignore": level != "reduced", "sensors": sorted(sensors)} # reduced visibility: average data, not greyed
|
||||
return {**e, "level": level} if level else e
|
||||
|
||||
|
||||
def _levels(measurements, instants):
|
||||
"""t -> "none" if at least HALF of the usually seen targets are rejected at t, "reduced" otherwise."""
|
||||
ratios = []
|
||||
for v in measurements.values():
|
||||
r = {m["t"]: m["error_value"] / m["error_threshold"] for m in v if m["error_value"] is not None and m["error_threshold"]}
|
||||
if r and statistics.median(r.values()) < 1:
|
||||
ratios.append(r)
|
||||
return {t: "none" if sum(r.get(t, 0) > 1 for r in ratios) >= HALF * max(1, sum(t in r for r in ratios)) else "reduced"
|
||||
for t in instants}
|
||||
|
||||
|
||||
def episodes(measurements):
|
||||
"""All error_value episodes: Visibility (Poor visibility), Obstruction (Brief obstruction, Target obscured)."""
|
||||
step = _step(measurements)
|
||||
gap = GAP_STEPS * step
|
||||
refs = {s for s, v in measurements.items() if v[0]["pattern_type"] == "global_ref"}
|
||||
instants = sorted({m["t"] for v in measurements.values() for m in v})
|
||||
high, rejections = {}, {} # sensor -> [t]
|
||||
refused = set() # instants where a target (not a global_ref) is above its threshold
|
||||
rejected_targets = {} # t -> {targets above their threshold}
|
||||
strong, refusals = {}, {} # t -> {sensors rejected at more than FACTOR times their median}, {sensors rejected}
|
||||
for s, v in measurements.items():
|
||||
e = [m["error_value"] for m in v if m["error_value"] is not None]
|
||||
if len(e) < 10:
|
||||
continue
|
||||
med = statistics.median(e)
|
||||
mad = statistics.median(abs(x - med) for x in e) or 1e-9
|
||||
for m in v:
|
||||
x = m["error_value"]
|
||||
if x is None:
|
||||
continue
|
||||
rejected = m["error_threshold"] and x > m["error_threshold"]
|
||||
if rejected and s not in refs:
|
||||
refused.add(m["t"])
|
||||
rejected_targets.setdefault(m["t"], set()).add(s)
|
||||
if x > med + K_MAD * mad and (x > FACTOR * med or rejected and x > REJECT_FACTOR * med):
|
||||
high.setdefault(s, []).append(m["t"])
|
||||
if rejected:
|
||||
refusals.setdefault(m["t"], set()).add(s)
|
||||
if x > FACTOR * med:
|
||||
strong.setdefault(m["t"], set()).add(s)
|
||||
if s not in refs and rejected and x > med + K_TARGET * mad:
|
||||
rejections.setdefault(s, []).append(m["t"])
|
||||
by_t = {} # t -> {high sensors}
|
||||
for s, ts in high.items():
|
||||
for t in ts:
|
||||
by_t.setdefault(t, set()).add(s)
|
||||
n = lambda t: len(by_t.get(t, ())) # noqa: E731
|
||||
out = []
|
||||
|
||||
# 2. Brief obstruction: an instant where many sensors are high, at least SALIENCE times more than the neighbours.
|
||||
briefs = set()
|
||||
for i, t in enumerate(instants):
|
||||
neighbours = [n(instants[j]) for j in (i - 1, i + 1) if 0 <= j < len(instants)]
|
||||
if n(t) >= BRIEF_MIN and n(t) >= SALIENCE * max(neighbours):
|
||||
briefs.add(t)
|
||||
only = rejected_targets.get(t, set()).difference(*(rejected_targets.get(instants[j], set()) for j in (i - 1, i + 1)
|
||||
if 0 <= j < len(instants)))
|
||||
if by_t[t] & refs and len(only) < BRIEF_MIN: # references touched without targets rejected at this instant
|
||||
continue # alone: not an obstacle in front of targets, the whole image
|
||||
out.append(_episode(t, t, "Brief obstruction", f"{n(t)} points were poorly seen for a single measurement, then came "
|
||||
"back: probably a train, a vehicle or an animal passing; ignore this point on the curves.", by_t[t]))
|
||||
elif 0 < len(f := strong.get(t, set())) < BRIEF_MIN and n(t) < BRIEF_MIN and len(f & refs) <= 1 and not f & set().union(
|
||||
*(refusals.get(instants[j], set()) for j in (i - 1, i + 1) if 0 <= j < len(instants))):
|
||||
# one or two targets, or a reference, rejected at this single measurement, at FACTOR times their median
|
||||
out.append(_episode(t, t, "Brief obstruction", f"{len(f)} point{'s were' if len(f) > 1 else ' was'} poorly seen "
|
||||
"for a single measurement, then came back: probably something passing; ignore this point on the "
|
||||
"curves.", f))
|
||||
|
||||
# 3. Poor visibility: long runs per sensor (without the brief instants), joined across sensors, then reduced to the
|
||||
# instants where at least FRAC of the episode's peak is high (cuts the tails of one or two sensors and the long
|
||||
# lulls, more than BREAK).
|
||||
runs = []
|
||||
for s, ts in high.items():
|
||||
pts = [(t, s) for t in ts if t not in briefs]
|
||||
runs += [(a, b, s) for a, b, _, l in _group(pts, gap) if len(l) >= RUN_MIN]
|
||||
zones = [z for z in _group_intervals(sorted(runs), gap) if len(z[2]) >= 2]
|
||||
kept = []
|
||||
for a, b, _ in zones:
|
||||
inside = [t for t in instants if a <= t <= b and t not in briefs]
|
||||
peak = max(n(t) for t in inside)
|
||||
pts = [(t, s) for t in inside if n(t) >= max(1, FRAC * peak) for s in by_t[t]]
|
||||
kept += [(d, f, cs) for d, f, cs, l in _group(pts, max(gap, BREAK)) if len(cs) >= 2 and len(set(l)) >= RUN_MIN]
|
||||
# Bounds: from the first to the last rejected target; without a rejection, no episode (6).
|
||||
kept = [(r[0], r[-1], cs) for a, b, cs in kept if (r := [t for t in instants if a <= t <= b and t in refused])]
|
||||
level = _levels(measurements, instants)
|
||||
for a, b, cs in kept: # one episode per period of the same level (5); the text comes from categories.py
|
||||
if len(cs) <= FEW and not cs & refs: # an object in front of a few targets, not a diffuse nuisance (7)
|
||||
out.append(_episode(a, b, "Target obscured", "", cs))
|
||||
continue
|
||||
seg = []
|
||||
for t in [t for t in instants if a <= t <= b] + [None]:
|
||||
if seg and (t is None or level[t] != level[seg[0]]):
|
||||
who = cs & set().union(*(by_t.get(u, set()) for u in seg)) or cs
|
||||
out.append(_episode(seg[0], seg[-1], "Poor visibility", "", who, level[seg[0]]))
|
||||
seg = []
|
||||
if t is not None:
|
||||
seg.append(t)
|
||||
inside_any = lambda t: t in briefs or any(a <= t <= b for a, b, _ in kept) # noqa: E731
|
||||
|
||||
# 4. Target obscured: a target rejected again and again, mostly outside the episodes above.
|
||||
targets = []
|
||||
for s, ts in rejections.items():
|
||||
for a, b, _, l in _group([(t, s) for t in ts], max(TARGET_GAP, 2 * step)):
|
||||
if len(l) >= (TARGET_MIN if TARGET_MIN * step <= TARGET_GAP else 2) and sum(map(inside_any, l)) * 2 < len(l):
|
||||
targets.append((a, b, s))
|
||||
for a, b, cs in _group_intervals(sorted(targets), timedelta(0)):
|
||||
who, it, its = ("One target was", "it", "its") if len(cs) == 1 else (f"{len(cs)} targets were", "them", "their")
|
||||
out.append(_episode(a, b, "Target obscured", f"{who} repeatedly lost while the targets around stayed clear: probably "
|
||||
f"something in front of {it} (leaves, snow) or glare; {its} displacements in this period are unreliable.", cs))
|
||||
return sorted(out, key=lambda e: e["start"])
|
||||
|
||||
|
||||
def diffuse(measurements):
|
||||
"""Diffuse nuisance (8): the runs of at least RUN_MIN instants where the median, over the sensors, of the error_value
|
||||
in MAD above each one's median reaches K_DIFFUSE, with or without rejections -> Poor visibility "reduced"."""
|
||||
z = {} # t -> {sensor: gap in MAD}
|
||||
for s, v in measurements.items():
|
||||
e = [m["error_value"] for m in v if m["error_value"] is not None]
|
||||
if len(e) < 10:
|
||||
continue
|
||||
med = statistics.median(e)
|
||||
mad = statistics.median(abs(x - med) for x in e) or 1e-9
|
||||
for m in v:
|
||||
if m["error_value"] is not None:
|
||||
z.setdefault(m["t"], {})[s] = (m["error_value"] - med) / mad
|
||||
highs = [(t, None) for t in sorted(z) if statistics.median(z[t].values()) >= K_DIFFUSE]
|
||||
return [_episode(a, b, "Poor visibility", "", {s for t in l for s, x in z[t].items() if x >= K_DIFFUSE}, "reduced")
|
||||
for a, b, _, l in _group(highs, GAP_STEPS * _step(measurements)) if len(l) >= RUN_MIN]
|
||||
|
||||
|
||||
def detect(measurements):
|
||||
shimmer = heat_shimmer(measurements) # a diffuse nuisance during heat shimmer is the heat shimmer (8)
|
||||
return ([e for e in episodes(measurements) if e["kind"] == "Visibility"] + shimmer
|
||||
+ [e for e in diffuse(measurements) if not any(t["start"] <= e["start"] and e["end"] <= t["end"] for t in shimmer)])
|
||||
|
||||
|
||||
# --- Heat shimmer ----------------------------------------------------------------------------------------------------
|
||||
# Atmospheric turbulence (fast noise on the corrected displacements). Method:
|
||||
# 1. Per sensor and per hour H, on the measurements of [H − 1 h, H + 2 h): agitation = median of |Δcorr_x| + |Δcorr_y|
|
||||
# between successive measurements (robust to one-step jumps), and median of the error_value.
|
||||
# 2. Each is divided by the sensor's calm: the lower quartile of its hourly values over the export (often the night).
|
||||
# 3. Turbulent hour: at least SHARE of the sensors at ≥ RATIO times their calm agitation, and median error_value of the
|
||||
# sensors at ≥ RATIO_ERR times the calm (warm air blurs the spots: E-003).
|
||||
# 4. Neighbouring hours merged (a one-hour hole tolerated), episode of at least MIN_DURATION, dropped if more than
|
||||
# REJECT_MAX of the measurements are rejected beyond their sensor's habit (its share of rejections over the export):
|
||||
# it is then poor visibility, another rule. A target that is always rejected (Target issue) does not count (learner
|
||||
# session Visibility, mod. 1, S1: WILDSTRUBEL, R3 rejected all day, the days of heat shimmer were dropped).
|
||||
# Greyed if the median agitation ≥ STRONG.
|
||||
RATIO = 1.5 # tuned on E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night)
|
||||
SHARE = 0.5 # share of the sensors agitated at the same time
|
||||
RATIO_ERR = 1.5 # tuned on E-003 (error_value ×1.7 to ×3.2 the night; ≈ ×1 at noon elsewhere)
|
||||
REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; rejections beyond the habit since learner
|
||||
# session Visibility, mod. 1, S1 (WILDSTRUBEL: 4 to 14 %, against 9 to 22 % raw)
|
||||
STRONG = 2.0 # E-003 (×2.3 to ×2.6) is strong; an episode between ×1.5 and ×2 stays visible
|
||||
MIN_DURATION = timedelta(hours=3) # E-003 lasts 10-11 h; drops the morning of the real movement of E-004 (2 h), which
|
||||
# the classification drops too, whatever the duration
|
||||
HOUR = timedelta(hours=1)
|
||||
|
||||
|
||||
def _hour(t):
|
||||
return t.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _ratios(points):
|
||||
"""[(t, value)] -> {hour H: median over [H − 1 h, H + 2 h) / lower quartile of these medians}."""
|
||||
by = {}
|
||||
for t, v in points:
|
||||
for k in (-1, 0, 1):
|
||||
by.setdefault(_hour(t) + k * HOUR, []).append(v)
|
||||
med = {h: statistics.median(v) for h, v in by.items() if len(v) >= 3}
|
||||
if len(med) < 8:
|
||||
return {}
|
||||
calm = statistics.quantiles(med.values(), n=4)[0]
|
||||
return {h: v / calm for h, v in med.items()} if calm > 0 else {}
|
||||
|
||||
|
||||
def hours(measurements):
|
||||
"""-> {hour: {sensor: (agitation / calm, error_value / calm)}}."""
|
||||
out = {}
|
||||
for s, ms in measurements.items():
|
||||
ag = _ratios([(b["t"], abs(b["corr_x"] - a["corr_x"]) + abs(b["corr_y"] - a["corr_y"]))
|
||||
for a, b in zip(ms, ms[1:]) if a["corr_x"] is not None and b["corr_x"] is not None])
|
||||
er = _ratios([(m["t"], m["error_value"]) for m in ms if m["error_value"] is not None])
|
||||
for h, r in ag.items():
|
||||
if h in er:
|
||||
out.setdefault(h, {})[s] = (r, er[h])
|
||||
return out
|
||||
|
||||
|
||||
def _turbulent(r):
|
||||
return (sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
|
||||
and statistics.median(e for _, e in r.values()) >= RATIO_ERR)
|
||||
|
||||
|
||||
def heat_shimmer(measurements):
|
||||
H = hours(measurements)
|
||||
blocks = []
|
||||
for h in sorted(h for h, r in H.items() if _turbulent(r)):
|
||||
if blocks and h - blocks[-1][-1] <= 2 * HOUR:
|
||||
blocks[-1].append(h)
|
||||
else:
|
||||
blocks.append([h])
|
||||
habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()} # usual rejections
|
||||
episodes = []
|
||||
for b in blocks:
|
||||
start, end = b[0], b[-1] + HOUR
|
||||
by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()}
|
||||
new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items())
|
||||
if end - start < MIN_DURATION or new > REJECT_MAX * sum(map(len, by.values())):
|
||||
continue
|
||||
by_sensor = {}
|
||||
for h in b:
|
||||
for s, (a, _) in H[h].items():
|
||||
by_sensor.setdefault(s, []).append(a)
|
||||
agit = {s: statistics.median(v) for s, v in by_sensor.items()}
|
||||
strength = statistics.median(agit.values())
|
||||
episodes.append({
|
||||
"start": iso(start), "end": iso(end), "kind": "Visibility",
|
||||
"label": "Heat shimmer",
|
||||
"text": f"Readings were about {strength:.1f}× noisier than in calm hours because of heat shimmer "
|
||||
"(warm air trembling in the line of sight); the small fast wiggles are not movement.",
|
||||
"ignore": strength >= STRONG,
|
||||
"sensors": sorted(s for s, v in agit.items() if v >= RATIO),
|
||||
})
|
||||
return episodes
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for e in detect(load(sys.argv[1])):
|
||||
print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])
|
||||
Reference in new issue
Block a user