2026-11-16 16:28:52 +01:00
|
|
|
|
"""Visibility family: the camera sees the targets less well (error_value), or the air trembles (heat shimmer). The same
|
|
|
|
|
|
error_value also finds objects in front of a few targets: Obstruction family (Brief obstruction, Target obscured).
|
|
|
|
|
|
|
|
|
|
|
|
1. High measurement: error_value above the median of its sensor + K_MAD MAD and above FACTOR times that median.
|
|
|
|
|
|
2. Brief obstruction (one instant): at least BRIEF_MIN sensors high, SALIENCE times more than at each neighbouring instant
|
|
|
|
|
|
(a train, a vehicle); or fewer than BRIEF_MIN sensors, at most one reference, rejected at FACTOR times their median
|
|
|
|
|
|
and not at the neighbouring instants (something passing in front of one or two targets).
|
|
|
|
|
|
3. Poor visibility: runs of at least RUN_MIN high measurements of a sensor (steps of at most GAP_STEPS), outside the brief
|
|
|
|
|
|
instants, overlapping on at least two sensors. Level of each instant: "none" (greyed) when at least HALF of the
|
|
|
|
|
|
usually seen targets are rejected, "reduced" otherwise; one episode per period of the same level.
|
|
|
|
|
|
4. Diffuse nuisance: at least RUN_MIN instants in a row where the median over the sensors of the error_value, in MAD
|
|
|
|
|
|
above each one's median, reaches K_DIFFUSE: Poor visibility "reduced", unless within heat shimmer or another Poor
|
|
|
|
|
|
visibility.
|
|
|
|
|
|
5. Target obscured: a target rejected above K_TARGET MAD at least TARGET_MIN times, less than TARGET_GAP apart, mostly
|
|
|
|
|
|
outside the episodes above (leaves, snow, glare on one target).
|
|
|
|
|
|
Heat shimmer: end of file.
|
2026-11-13 12:19:52 +01:00
|
|
|
|
"""
|
|
|
|
|
|
import statistics
|
|
|
|
|
|
from datetime import timedelta
|
|
|
|
|
|
|
2026-11-16 16:28:52 +01:00
|
|
|
|
from common import cadence, episode, level, runs, targets
|
|
|
|
|
|
from data import is_reference
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
2026-11-16 16:28:52 +01:00
|
|
|
|
K_MAD = 10 # E-001 (fog: 11 to 57 MAD), E-005 (rain, trains: 10 to 70 MAD)
|
|
|
|
|
|
FACTOR = 2 # drops the sensors with a near-zero MAD (E-004, E-005)
|
|
|
|
|
|
RUN_MIN = 3 # E-001, rain of E-005
|
|
|
|
|
|
GAP_STEPS = 2.5 # steps tolerated inside a run (a missing step, timestamp jitter)
|
|
|
|
|
|
BRIEF_MIN = 3 # trains of E-005: 5 to 17 sensors
|
|
|
|
|
|
SALIENCE = 3 # a train stands out (E-005: 5 to 17 against 0 to 2); the cobweb of E-002 varies from 2 to 9
|
|
|
|
|
|
HALF = 0.5 # "nothing is seen": NORDEND 09.10 23:00 (68 % rejected) against the rain of E-005 (24 to 33 %)
|
|
|
|
|
|
K_DIFFUSE = 2.5 # wf-improve Visibility, mod. 1, S2 (JUNGFRAU, FLORIDA, NORDEND: the whole scene less clear)
|
|
|
|
|
|
K_TARGET = 5 # leaves of E-005: 6 to 25 MAD, rejected
|
|
|
|
|
|
TARGET_MIN = 3
|
|
|
|
|
|
TARGET_GAP = timedelta(hours=6) # leaves of E-005: 7 rejections in 12 h, up to 5 h apart
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
2026-11-16 16:28:52 +01:00
|
|
|
|
BRIEF = "Something passed in front of {} for a single measurement. Ignore this point on the curves."
|
|
|
|
|
|
OBSCURED = "{} was hidden several times while the targets around it stayed clear. Ignore its curve in this period."
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _levels(measurements, instants):
|
|
|
|
|
|
"""t -> "none" if at least HALF of the usually seen targets are rejected at t, "reduced" otherwise."""
|
|
|
|
|
|
ratios = []
|
|
|
|
|
|
for v in measurements.values():
|
|
|
|
|
|
r = {m["t"]: m["error_value"] / m["error_threshold"] for m in v if m["error_value"] is not None and m["error_threshold"]}
|
|
|
|
|
|
if r and statistics.median(r.values()) < 1:
|
|
|
|
|
|
ratios.append(r)
|
|
|
|
|
|
return {t: "none" if sum(r.get(t, 0) > 1 for r in ratios) >= HALF * max(1, sum(t in r for r in ratios)) else "reduced"
|
|
|
|
|
|
for t in instants}
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 16:28:52 +01:00
|
|
|
|
def _poor(start, end, sensors, level, refs):
|
|
|
|
|
|
refs = ", including the reference targets" if set(sensors) & refs else ""
|
|
|
|
|
|
text = (f"The camera could not see {targets(len(sensors))} clearly{refs}. The curves in this period are not "
|
|
|
|
|
|
"reliable: ignore them." if level == "none" else f"The camera saw {targets(len(sensors))} less clearly than "
|
|
|
|
|
|
f"usual{refs}. The data in this period are less precise: use them with care.")
|
|
|
|
|
|
return episode(start, end, "Visibility", "Poor visibility", text, sensors, level == "none", level=level)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def detect(measurements):
|
|
|
|
|
|
gap = GAP_STEPS * cadence(measurements)
|
|
|
|
|
|
refs = {s for s, v in measurements.items() if is_reference(v)}
|
2026-11-13 12:19:52 +01:00
|
|
|
|
instants = sorted({m["t"] for v in measurements.values() for m in v})
|
2026-11-16 16:28:52 +01:00
|
|
|
|
z = {} # t -> {sensor: error_value in MAD above the sensor's median}
|
|
|
|
|
|
high, strong, rejected, repeated = {}, {}, {}, {} # t -> {sensors}; repeated: target -> [t]
|
2026-11-13 12:19:52 +01:00
|
|
|
|
for s, v in measurements.items():
|
|
|
|
|
|
e = [m["error_value"] for m in v if m["error_value"] is not None]
|
|
|
|
|
|
if len(e) < 10:
|
|
|
|
|
|
continue
|
2026-11-16 16:28:52 +01:00
|
|
|
|
med, mad = level(e)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
for m in v:
|
2026-11-16 16:28:52 +01:00
|
|
|
|
t, x = m["t"], m["error_value"]
|
2026-11-13 12:19:52 +01:00
|
|
|
|
if x is None:
|
|
|
|
|
|
continue
|
2026-11-16 16:28:52 +01:00
|
|
|
|
z.setdefault(t, {})[s] = (x - med) / mad
|
|
|
|
|
|
if x > med + K_MAD * mad and x > FACTOR * med:
|
|
|
|
|
|
high.setdefault(t, set()).add(s)
|
|
|
|
|
|
if m["error_threshold"] and x > m["error_threshold"]:
|
|
|
|
|
|
rejected.setdefault(t, set()).add(s)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
if x > FACTOR * med:
|
2026-11-16 16:28:52 +01:00
|
|
|
|
strong.setdefault(t, set()).add(s)
|
|
|
|
|
|
if s not in refs and x > med + K_TARGET * mad:
|
|
|
|
|
|
repeated.setdefault(s, []).append(t)
|
|
|
|
|
|
n = lambda t: len(high.get(t, ())) # noqa: E731
|
|
|
|
|
|
out, briefs = [], set()
|
|
|
|
|
|
|
|
|
|
|
|
for i, t in enumerate(instants): # 2. Brief obstruction
|
|
|
|
|
|
around = [instants[j] for j in (i - 1, i + 1) if 0 <= j < len(instants)]
|
|
|
|
|
|
few = strong.get(t, set())
|
|
|
|
|
|
if n(t) >= BRIEF_MIN and n(t) >= SALIENCE * max(map(n, around), default=0):
|
2026-11-13 12:19:52 +01:00
|
|
|
|
briefs.add(t)
|
2026-11-16 16:28:52 +01:00
|
|
|
|
few = high[t]
|
|
|
|
|
|
elif not (few and len(few) < BRIEF_MIN and n(t) < BRIEF_MIN and len(few & refs) <= 1
|
|
|
|
|
|
and not few & set().union(*(rejected.get(u, set()) for u in around))):
|
2026-11-13 12:19:52 +01:00
|
|
|
|
continue
|
2026-11-16 16:28:52 +01:00
|
|
|
|
out.append(episode(t, t, "Obstruction", "Brief obstruction", BRIEF.format(targets(len(few))), few, True))
|
|
|
|
|
|
|
|
|
|
|
|
spans = [] # 3. Poor visibility: runs of each sensor, then the overlapping runs of at least two sensors
|
|
|
|
|
|
for s in {s for t in high for s in high[t]}:
|
|
|
|
|
|
spans += [(r[0], r[-1], s) for r in runs([t for t in instants if s in high.get(t, ()) and t not in briefs], gap)
|
|
|
|
|
|
if len(r) >= RUN_MIN]
|
|
|
|
|
|
zones = []
|
|
|
|
|
|
for a, b, s in sorted(spans):
|
|
|
|
|
|
if zones and a - zones[-1][1] <= gap:
|
|
|
|
|
|
zones[-1] = (zones[-1][0], max(zones[-1][1], b), zones[-1][2] | {s})
|
|
|
|
|
|
else:
|
|
|
|
|
|
zones.append((a, b, {s}))
|
|
|
|
|
|
zones = [zone for zone in zones if len(zone[2]) >= 2]
|
|
|
|
|
|
lv = _levels(measurements, instants)
|
|
|
|
|
|
for a, b, cs in zones:
|
|
|
|
|
|
inside = [t for t in instants if a <= t <= b]
|
|
|
|
|
|
for k, t in enumerate(inside):
|
|
|
|
|
|
seg = [t] if k == 0 or lv[t] != lv[inside[k - 1]] else seg + [t]
|
|
|
|
|
|
if k == len(inside) - 1 or lv[inside[k + 1]] != lv[t]:
|
|
|
|
|
|
out.append(_poor(seg[0], seg[-1], cs & set().union(*(high.get(u, set()) for u in seg)) or cs, lv[t], refs))
|
|
|
|
|
|
|
|
|
|
|
|
shimmer = heat_shimmer(measurements) # 4. diffuse nuisance
|
|
|
|
|
|
hosts = shimmer + [e for e in out if e["kind"] == "Visibility"]
|
|
|
|
|
|
for r in runs([t for t in sorted(z) if statistics.median(z[t].values()) >= K_DIFFUSE], gap):
|
|
|
|
|
|
e = _poor(r[0], r[-1], {s for t in r for s, x in z[t].items() if x >= K_DIFFUSE}, "reduced", refs)
|
|
|
|
|
|
if len(r) >= RUN_MIN and not any(h["start"] <= e["start"] and e["end"] <= h["end"] for h in hosts):
|
|
|
|
|
|
out.append(e)
|
|
|
|
|
|
|
|
|
|
|
|
seen = lambda t: t in briefs or any(a <= t <= b for a, b, _ in zones) # noqa: E731
|
|
|
|
|
|
for s, ts in repeated.items(): # 5. Target obscured
|
|
|
|
|
|
for r in runs(ts, TARGET_GAP):
|
|
|
|
|
|
if len(r) >= TARGET_MIN and 2 * sum(map(seen, r)) < len(r):
|
|
|
|
|
|
out.append(episode(r[0], r[-1], "Obstruction", "Target obscured", OBSCURED.format(s), [s], True))
|
|
|
|
|
|
return out + shimmer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --- Heat shimmer: fast noise of the corrected displacements in warm air.
|
|
|
|
|
|
# 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive
|
|
|
|
|
|
# measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values).
|
|
|
|
|
|
# 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO.
|
|
|
|
|
|
# 3. Turbulent hours at most 2 h apart make an episode, dropped if more than REJECT_MAX of its measurements are rejected
|
|
|
|
|
|
# beyond each sensor's habit (its share of rejections over the export): that is poor visibility. Greyed if the median
|
|
|
|
|
|
# agitation reaches STRONG.
|
|
|
|
|
|
RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2)
|
|
|
|
|
|
SHARE = 0.5
|
|
|
|
|
|
REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; beyond the habit: wf-improve Visibility, mod. 1, S1
|
|
|
|
|
|
STRONG = 2.0 # E-003 (×2.3 to ×2.6)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
HOUR = timedelta(hours=1)
|
2026-11-16 16:28:52 +01:00
|
|
|
|
SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement."
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _ratios(points):
|
2026-11-16 16:28:52 +01:00
|
|
|
|
"""[(t, value)] -> {hour H: median over [H - 1 h, H + 2 h) / lower quartile of these medians}."""
|
2026-11-13 12:19:52 +01:00
|
|
|
|
by = {}
|
|
|
|
|
|
for t, v in points:
|
|
|
|
|
|
for k in (-1, 0, 1):
|
2026-11-16 16:28:52 +01:00
|
|
|
|
by.setdefault(t.replace(minute=0, second=0, microsecond=0) + k * HOUR, []).append(v)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
med = {h: statistics.median(v) for h, v in by.items() if len(v) >= 3}
|
2026-11-16 16:28:52 +01:00
|
|
|
|
calm = statistics.quantiles(med.values(), n=4)[0] if len(med) >= 8 else 0
|
2026-11-13 12:19:52 +01:00
|
|
|
|
return {h: v / calm for h, v in med.items()} if calm > 0 else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 16:28:52 +01:00
|
|
|
|
def heat_shimmer(measurements):
|
|
|
|
|
|
hours = {} # hour -> {sensor: (agitation / calm, error_value / calm)}
|
2026-11-13 12:19:52 +01:00
|
|
|
|
for s, ms in measurements.items():
|
|
|
|
|
|
ag = _ratios([(b["t"], abs(b["corr_x"] - a["corr_x"]) + abs(b["corr_y"] - a["corr_y"]))
|
|
|
|
|
|
for a, b in zip(ms, ms[1:]) if a["corr_x"] is not None and b["corr_x"] is not None])
|
|
|
|
|
|
er = _ratios([(m["t"], m["error_value"]) for m in ms if m["error_value"] is not None])
|
2026-11-16 16:28:52 +01:00
|
|
|
|
for h in ag.keys() & er.keys():
|
|
|
|
|
|
hours.setdefault(h, {})[s] = (ag[h], er[h])
|
|
|
|
|
|
turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
|
|
|
|
|
|
and statistics.median(e for _, e in r.values()) >= RATIO]
|
|
|
|
|
|
habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()}
|
|
|
|
|
|
out = []
|
|
|
|
|
|
for b in runs(turbulent, 2 * HOUR):
|
2026-11-13 12:19:52 +01:00
|
|
|
|
start, end = b[0], b[-1] + HOUR
|
|
|
|
|
|
by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()}
|
|
|
|
|
|
new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items())
|
2026-11-16 16:28:52 +01:00
|
|
|
|
if new > REJECT_MAX * sum(map(len, by.values())):
|
2026-11-13 12:19:52 +01:00
|
|
|
|
continue
|
2026-11-16 16:28:52 +01:00
|
|
|
|
agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}}
|
2026-11-13 12:19:52 +01:00
|
|
|
|
strength = statistics.median(agit.values())
|
2026-11-16 16:28:52 +01:00
|
|
|
|
out.append(episode(start, end, "Visibility", "Heat shimmer", SHIMMER.format(strength),
|
|
|
|
|
|
[s for s, v in agit.items() if v >= RATIO], strength >= STRONG))
|
|
|
|
|
|
return out
|