REJECT_MAX of heat shimmer and CALM of target_issue removed

Heat shimmer is no longer dropped for its rejections beyond each sensor's habit (Visibility found +2: WILDSTRUBEL +7,
NIEDERHORN +4). target_issue counts every measurement of the day, not only the calm instants (Obstruction found +3,
correct -2: correct -11 at LENZSPITZE and -12 at WILDSTRUBEL; Target issue unchanged).

Benchmark Totals (from the previous commit): Visibility found 55 -> 57 %; Obstruction found 42 -> 45 %, correct
33 -> 31 %; the other families unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 17:28:59 +01:00
1 parent 4188efe201
commit ab0347c41d
2 files changed
+18 -38

No files matched your search

+16 -27
View File
@@ -1,19 +1,17 @@
"""Target issue family: a target rejected day after day while the others are seen ("Degraded target"). visibility.py """Target issue family: a target rejected day after day ("Degraded target"); visibility.py compares each sensor with its
compares each sensor with its own usual level, so a target almost always rejected looks normal to it. own usual level, so a target almost always rejected looks normal to it. For a few days only, Target obscured
(Obstruction family).
1. Calm instant for a sensor: at most CALM of the other sensors rejected (weather and fog, which touch everyone, are set 1. Degraded day: at least SHARE of the sensor's measurements of the UTC day rejected; a day with fewer than MIN_HOURS of
aside). measurements (at the station's cadence) does not count.
2. Degraded day: at least SHARE of the sensor's calm instants of the UTC day rejected; a day with fewer than MIN_HOURS of 2. Run: degraded days with at most BRIDGE other days in a row between them, and SHARE of rejections over the run. WEEK
calm instants (at the station's cadence) does not count. days or more: Degraded target; MIN_DAYS or more: Target obscured. The episode runs from the first rejected
3. Problem target: degraded days from a first to a last at least WEEK days apart, at most BRIDGE non-degraded days in a measurement of the run to the last, greyed. The cause (dirt, vegetation, a parked vehicle) is seen on the images.
row between them, and SHARE of rejections over these days; the episode runs from its first rejected measurement to
its last, greyed. The cause (dirt, vegetation, setting) is only seen on the images.
""" """
from datetime import timedelta from datetime import timedelta
from common import cadence, episode from common import cadence, episode
CALM = 0.25 # share of the other sensors rejected, at most, at a calm instant
SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %) SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %)
MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants) MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants)
BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day) BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day)
@@ -26,14 +24,14 @@ def rejected(m):
def _runs(days, min_instants): def _runs(days, min_instants):
"""{day: [rejected, calm, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two """{day: [rejected, measurements, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two
degraded days belong to the run; a day that does not count is skipped.""" degraded days belong to the run; a day that does not count is skipped."""
runs, cur, gap = [], [], [] runs, cur, gap = [], [], []
for d in sorted(days): for d in sorted(days):
rej, calm = days[d][:2] rej, n = days[d][:2]
if calm < min_instants: if n < min_instants:
continue continue
if rej / calm >= SHARE: if rej / n >= SHARE:
cur += gap + [days[d]] cur += gap + [days[d]]
gap = [] gap = []
elif cur: elif cur:
@@ -46,19 +44,11 @@ def _runs(days, min_instants):
def detect(measurements): def detect(measurements):
min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements) min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements)
by_t = {} days = {} # sensor -> day -> [rejected, measurements, first, last]
for s, ms in measurements.items(): for s, ms in measurements.items():
for m in ms: for m in ms:
by_t.setdefault(m["t"], {})[s] = rejected(m) c = days.setdefault(s, {}).setdefault(m["t"].date(), [0, 0, m["t"], m["t"]])
days = {} # sensor -> day -> [rejected, calm, first measurement, last] c[0], c[1], c[3] = c[0] + rejected(m), c[1] + 1, m["t"]
for t in sorted(by_t):
for s, r in by_t[t].items():
others = [v for k, v in by_t[t].items() if k != s]
c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t])
c[3] = t
if others and sum(others) / len(others) <= CALM:
c[0] += r
c[1] += 1
out = [] out = []
for s, ds in sorted(days.items()): for s, ds in sorted(days.items()):
for run in _runs(ds, min_instants): for run in _runs(ds, min_instants):
@@ -67,8 +57,7 @@ def detect(measurements):
continue continue
bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)] bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)]
n = sum(c[0] / c[1] >= SHARE for c in run) n = sum(c[0] / c[1] >= SHARE for c in run)
seen = f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were seen, on " \ seen = f"{s} was rejected in {round(100 * share)} % of the measurements, on {n} of {len(run)} days."
f"{n} of {len(run)} days."
out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period " out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period "
"and look at the images of the target.", [s], True) if len(run) >= WEEK else "and look at the images of the target.", [s], True) if len(run) >= WEEK else
episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few " episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few "
+2 -11
View File
@@ -132,12 +132,9 @@ def detect(measurements):
# 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive # 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive
# measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values). # measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values).
# 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO. # 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO.
# 3. Turbulent hours at most 2 h apart make an episode, dropped if more than REJECT_MAX of its measurements are rejected # 3. Turbulent hours at most 2 h apart make an episode, greyed if the median agitation reaches STRONG.
# beyond each sensor's habit (its share of rejections over the export): that is poor visibility. Greyed if the median
# agitation reaches STRONG.
RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2) RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2)
SHARE = 0.5 SHARE = 0.5
REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; beyond the habit: wf-improve Visibility, mod. 1, S1
STRONG = 2.0 # E-003 (×2.3 to ×2.6) STRONG = 2.0 # E-003 (×2.3 to ×2.6)
HOUR = timedelta(hours=1) HOUR = timedelta(hours=1)
SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement." SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement."
@@ -164,16 +161,10 @@ def heat_shimmer(measurements):
hours.setdefault(h, {})[s] = (ag[h], er[h]) hours.setdefault(h, {})[s] = (ag[h], er[h])
turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r) turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
and statistics.median(e for _, e in r.values()) >= RATIO] and statistics.median(e for _, e in r.values()) >= RATIO]
habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()}
out = [] out = []
for b in runs(turbulent, 2 * HOUR): for b in runs(turbulent, 2 * HOUR):
start, end = b[0], b[-1] + HOUR
by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()}
new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items())
if new > REJECT_MAX * sum(map(len, by.values())):
continue
agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}} agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}}
strength = statistics.median(agit.values()) strength = statistics.median(agit.values())
out.append(episode(start, end, "Visibility", "Heat shimmer", SHIMMER.format(strength), out.append(episode(b[0], b[-1] + HOUR, "Visibility", "Heat shimmer", SHIMMER.format(strength),
[s for s, v in agit.items() if v >= RATIO], strength >= STRONG)) [s for s, v in agit.items() if v >= RATIO], strength >= STRONG))
return out return out