Files
aimsight-detectors/scripts/visibility.py
T

157 lines
8.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Visibility family: the camera sees the targets less well (error_value), or the air trembles (heat shimmer). The same
error_value also finds something passing in front of a few targets: Brief obstruction (Obstruction family).
1. High measurement: error_value above the median of its sensor + K_MAD MAD and above FACTOR times that median.
2. Brief obstruction (one instant): at least BRIEF_MIN sensors high, SALIENCE times more than at each neighbouring instant
(a train, a vehicle); or fewer than BRIEF_MIN sensors, at most one reference, rejected at FACTOR times their median
and not at the neighbouring instants (something passing in front of one or two targets).
3. Poor visibility: runs of at least RUN_MIN high measurements of a sensor (steps of at most GAP_STEPS), outside the brief
instants, overlapping on at least two sensors. Level of each instant: "none" (greyed) when at least HALF of the
usually seen targets are rejected, "reduced" otherwise; one episode per period of the same level.
4. Diffuse nuisance: at least RUN_MIN instants in a row where the median over the sensors of the error_value, in MAD
above each one's median, reaches K_DIFFUSE: Poor visibility "reduced", unless within heat shimmer or another Poor
visibility.
Heat shimmer: end of file.
"""
import statistics
from datetime import timedelta
from episodes import cadence, episode, level, runs, targets
from data import is_reference
K_MAD = 10 # E-001 (fog: 11 to 57 MAD), E-005 (rain, trains: 10 to 70 MAD)
FACTOR = 2 # drops the sensors with a near-zero MAD (E-004, E-005)
RUN_MIN = 3 # E-001, rain of E-005
GAP_STEPS = 2.5 # steps tolerated inside a run (a missing step, timestamp jitter)
BRIEF_MIN = 3 # trains of E-005: 5 to 17 sensors
SALIENCE = 3 # a train stands out (E-005: 5 to 17 against 0 to 2); the cobweb of E-002 varies from 2 to 9
HALF = 0.5 # "nothing is seen": NORDEND 09.10 23:00 (68 % rejected) against the rain of E-005 (24 to 33 %)
K_DIFFUSE = 2.5 # wf-improve Visibility, mod. 1, S2 (JUNGFRAU, FLORIDA, NORDEND: the whole scene less clear)
BRIEF = "Something passed in front of {} for a single measurement. Ignore this point on the curves."
def _levels(measurements, instants):
"""t -> "none" if at least HALF of the usually seen targets are rejected at t, "reduced" otherwise."""
ratios = []
for v in measurements.values():
r = {m["t"]: m["error_value"] / m["error_threshold"] for m in v if m["error_value"] is not None and m["error_threshold"]}
if r and statistics.median(r.values()) < 1:
ratios.append(r)
return {t: "none" if sum(r.get(t, 0) > 1 for r in ratios) >= HALF * max(1, sum(t in r for r in ratios)) else "reduced"
for t in instants}
def _poor(start, end, sensors, level, refs):
refs = ", including the reference targets" if set(sensors) & refs else ""
text = (f"The camera could not see {targets(len(sensors))} clearly{refs}. The curves in this period are not "
"reliable: ignore them." if level == "none" else f"The camera saw {targets(len(sensors))} less clearly than "
f"usual{refs}. The data in this period are less precise: use them with care.")
return episode(start, end, "Visibility", "Poor visibility", text, sensors, level == "none", level=level)
def detect(measurements):
gap = GAP_STEPS * cadence(measurements)
refs = {s for s, v in measurements.items() if is_reference(v)}
instants = sorted({m["t"] for v in measurements.values() for m in v})
z = {} # t -> {sensor: error_value in MAD above the sensor's median}
high, strong, rejected = {}, {}, {} # t -> {sensors}
for s, v in measurements.items():
e = [m["error_value"] for m in v if m["error_value"] is not None]
if len(e) < 10:
continue
med, mad = level(e)
for m in v:
t, x = m["t"], m["error_value"]
if x is None:
continue
z.setdefault(t, {})[s] = (x - med) / mad
if x > med + K_MAD * mad and x > FACTOR * med:
high.setdefault(t, set()).add(s)
if m["error_threshold"] and x > m["error_threshold"]:
rejected.setdefault(t, set()).add(s)
if x > FACTOR * med:
strong.setdefault(t, set()).add(s)
n = lambda t: len(high.get(t, ())) # noqa: E731
out, briefs = [], set()
for i, t in enumerate(instants): # 2. Brief obstruction
around = [instants[j] for j in (i - 1, i + 1) if 0 <= j < len(instants)]
few = strong.get(t, set())
if n(t) >= BRIEF_MIN and n(t) >= SALIENCE * max(map(n, around), default=0):
briefs.add(t)
few = high[t]
elif not (few and len(few) < BRIEF_MIN and n(t) < BRIEF_MIN and len(few & refs) <= 1
and not few & set().union(*(rejected.get(u, set()) for u in around))):
continue
out.append(episode(t, t, "Obstruction", "Brief obstruction", BRIEF.format(targets(len(few))), few, True))
spans = [] # 3. Poor visibility: runs of each sensor, then the overlapping runs of at least two sensors
for s in {s for t in high for s in high[t]}:
spans += [(r[0], r[-1], s) for r in runs([t for t in instants if s in high.get(t, ()) and t not in briefs], gap)
if len(r) >= RUN_MIN]
zones = []
for a, b, s in sorted(spans):
if zones and a - zones[-1][1] <= gap:
zones[-1] = (zones[-1][0], max(zones[-1][1], b), zones[-1][2] | {s})
else:
zones.append((a, b, {s}))
zones = [zone for zone in zones if len(zone[2]) >= 2]
lv = _levels(measurements, instants)
for a, b, cs in zones:
inside = [t for t in instants if a <= t <= b]
for k, t in enumerate(inside):
seg = [t] if k == 0 or lv[t] != lv[inside[k - 1]] else seg + [t]
if k == len(inside) - 1 or lv[inside[k + 1]] != lv[t]:
out.append(_poor(seg[0], seg[-1], cs & set().union(*(high.get(u, set()) for u in seg)) or cs, lv[t], refs))
shimmer = heat_shimmer(measurements) # 4. diffuse nuisance
hosts = shimmer + [e for e in out if e["kind"] == "Visibility"]
for r in runs([t for t in sorted(z) if statistics.median(z[t].values()) >= K_DIFFUSE], gap):
e = _poor(r[0], r[-1], {s for t in r for s, x in z[t].items() if x >= K_DIFFUSE}, "reduced", refs)
if len(r) >= RUN_MIN and not any(h["start"] <= e["start"] and e["end"] <= h["end"] for h in hosts):
out.append(e)
return out + shimmer
# --- Heat shimmer: fast noise of the corrected displacements in warm air.
# 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive
# measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values).
# 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO.
# 3. Turbulent hours at most 2 h apart make an episode, greyed if the median agitation reaches STRONG.
RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2)
SHARE = 0.5
STRONG = 2.0 # E-003 (×2.3 to ×2.6)
HOUR = timedelta(hours=1)
SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement."
def _ratios(points):
"""[(t, value)] -> {hour H: median over [H - 1 h, H + 2 h) / lower quartile of these medians}."""
by = {}
for t, v in points:
for k in (-1, 0, 1):
by.setdefault(t.replace(minute=0, second=0, microsecond=0) + k * HOUR, []).append(v)
med = {h: statistics.median(v) for h, v in by.items() if len(v) >= 3}
calm = statistics.quantiles(med.values(), n=4)[0] if len(med) >= 8 else 0
return {h: v / calm for h, v in med.items()} if calm > 0 else {}
def heat_shimmer(measurements):
hours = {} # hour -> {sensor: (agitation / calm, error_value / calm)}
for s, ms in measurements.items():
ag = _ratios([(b["t"], abs(b["corr_x"] - a["corr_x"]) + abs(b["corr_y"] - a["corr_y"]))
for a, b in zip(ms, ms[1:]) if a["corr_x"] is not None and b["corr_x"] is not None])
er = _ratios([(m["t"], m["error_value"]) for m in ms if m["error_value"] is not None])
for h in ag.keys() & er.keys():
hours.setdefault(h, {})[s] = (ag[h], er[h])
turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
and statistics.median(e for _, e in r.values()) >= RATIO]
out = []
for b in runs(turbulent, 2 * HOUR):
agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}}
strength = statistics.median(agit.values())
out.append(episode(b[0], b[-1] + HOUR, "Visibility", "Heat shimmer", SHIMMER.format(strength),
[s for s, v in agit.items() if v >= RATIO], strength >= STRONG))
return out