From ab0347c41d43c1dc1a9ac2545f572cae175af0b5 Mon Sep 17 00:00:00 2001 From: maxime Date: Mon, 16 Nov 2026 17:28:59 +0100 Subject: [PATCH] REJECT_MAX of heat shimmer and CALM of target_issue removed Heat shimmer is no longer dropped for its rejections beyond each sensor's habit (Visibility found +2: WILDSTRUBEL +7, NIEDERHORN +4). target_issue counts every measurement of the day, not only the calm instants (Obstruction found +3, correct -2: correct -11 at LENZSPITZE and -12 at WILDSTRUBEL; Target issue unchanged). Benchmark Totals (from the previous commit): Visibility found 55 -> 57 %; Obstruction found 42 -> 45 %, correct 33 -> 31 %; the other families unchanged. Co-Authored-By: Claude Opus 5.5 --- scripts/target_issue.py | 43 +++++++++++++++-------------------------- scripts/visibility.py | 13 ++----------- 2 files changed, 18 insertions(+), 38 deletions(-) diff --git a/scripts/target_issue.py b/scripts/target_issue.py index 1132197..0055d0c 100644 --- a/scripts/target_issue.py +++ b/scripts/target_issue.py @@ -1,19 +1,17 @@ -"""Target issue family: a target rejected day after day while the others are seen ("Degraded target"). visibility.py -compares each sensor with its own usual level, so a target almost always rejected looks normal to it. +"""Target issue family: a target rejected day after day ("Degraded target"); visibility.py compares each sensor with its +own usual level, so a target almost always rejected looks normal to it. For a few days only, Target obscured +(Obstruction family). -1. Calm instant for a sensor: at most CALM of the other sensors rejected (weather and fog, which touch everyone, are set - aside). -2. Degraded day: at least SHARE of the sensor's calm instants of the UTC day rejected; a day with fewer than MIN_HOURS of - calm instants (at the station's cadence) does not count. -3. Problem target: degraded days from a first to a last at least WEEK days apart, at most BRIDGE non-degraded days in a - row between them, and SHARE of rejections over these days; the episode runs from its first rejected measurement to - its last, greyed. The cause (dirt, vegetation, setting) is only seen on the images. +1. Degraded day: at least SHARE of the sensor's measurements of the UTC day rejected; a day with fewer than MIN_HOURS of + measurements (at the station's cadence) does not count. +2. Run: degraded days with at most BRIDGE other days in a row between them, and SHARE of rejections over the run. WEEK + days or more: Degraded target; MIN_DAYS or more: Target obscured. The episode runs from the first rejected + measurement of the run to the last, greyed. The cause (dirt, vegetation, a parked vehicle) is seen on the images. """ from datetime import timedelta from common import cadence, episode -CALM = 0.25 # share of the other sensors rejected, at most, at a calm instant SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %) MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants) BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day) @@ -26,14 +24,14 @@ def rejected(m): def _runs(days, min_instants): - """{day: [rejected, calm, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two + """{day: [rejected, measurements, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two degraded days belong to the run; a day that does not count is skipped.""" runs, cur, gap = [], [], [] for d in sorted(days): - rej, calm = days[d][:2] - if calm < min_instants: + rej, n = days[d][:2] + if n < min_instants: continue - if rej / calm >= SHARE: + if rej / n >= SHARE: cur += gap + [days[d]] gap = [] elif cur: @@ -46,19 +44,11 @@ def _runs(days, min_instants): def detect(measurements): min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements) - by_t = {} + days = {} # sensor -> day -> [rejected, measurements, first, last] for s, ms in measurements.items(): for m in ms: - by_t.setdefault(m["t"], {})[s] = rejected(m) - days = {} # sensor -> day -> [rejected, calm, first measurement, last] - for t in sorted(by_t): - for s, r in by_t[t].items(): - others = [v for k, v in by_t[t].items() if k != s] - c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t]) - c[3] = t - if others and sum(others) / len(others) <= CALM: - c[0] += r - c[1] += 1 + c = days.setdefault(s, {}).setdefault(m["t"].date(), [0, 0, m["t"], m["t"]]) + c[0], c[1], c[3] = c[0] + rejected(m), c[1] + 1, m["t"] out = [] for s, ds in sorted(days.items()): for run in _runs(ds, min_instants): @@ -67,8 +57,7 @@ def detect(measurements): continue bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)] n = sum(c[0] / c[1] >= SHARE for c in run) - seen = f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were seen, on " \ - f"{n} of {len(run)} days." + seen = f"{s} was rejected in {round(100 * share)} % of the measurements, on {n} of {len(run)} days." out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period " "and look at the images of the target.", [s], True) if len(run) >= WEEK else episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few " diff --git a/scripts/visibility.py b/scripts/visibility.py index ee44489..bfb91ee 100644 --- a/scripts/visibility.py +++ b/scripts/visibility.py @@ -132,12 +132,9 @@ def detect(measurements): # 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive # measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values). # 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO. -# 3. Turbulent hours at most 2 h apart make an episode, dropped if more than REJECT_MAX of its measurements are rejected -# beyond each sensor's habit (its share of rejections over the export): that is poor visibility. Greyed if the median -# agitation reaches STRONG. +# 3. Turbulent hours at most 2 h apart make an episode, greyed if the median agitation reaches STRONG. RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2) SHARE = 0.5 -REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; beyond the habit: wf-improve Visibility, mod. 1, S1 STRONG = 2.0 # E-003 (×2.3 to ×2.6) HOUR = timedelta(hours=1) SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement." @@ -164,16 +161,10 @@ def heat_shimmer(measurements): hours.setdefault(h, {})[s] = (ag[h], er[h]) turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r) and statistics.median(e for _, e in r.values()) >= RATIO] - habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()} out = [] for b in runs(turbulent, 2 * HOUR): - start, end = b[0], b[-1] + HOUR - by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()} - new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items()) - if new > REJECT_MAX * sum(map(len, by.values())): - continue agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}} strength = statistics.median(agit.values()) - out.append(episode(start, end, "Visibility", "Heat shimmer", SHIMMER.format(strength), + out.append(episode(b[0], b[-1] + HOUR, "Visibility", "Heat shimmer", SHIMMER.format(strength), [s for s, v in agit.items() if v >= RATIO], strength >= STRONG)) return out