REJECT_MAX of heat shimmer and CALM of target_issue removed
Heat shimmer is no longer dropped for its rejections beyond each sensor's habit (Visibility found +2: WILDSTRUBEL +7, NIEDERHORN +4). target_issue counts every measurement of the day, not only the calm instants (Obstruction found +3, correct -2: correct -11 at LENZSPITZE and -12 at WILDSTRUBEL; Target issue unchanged). Benchmark Totals (from the previous commit): Visibility found 55 -> 57 %; Obstruction found 42 -> 45 %, correct 33 -> 31 %; the other families unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4188efe201
commit
ab0347c41d
2 files changed
+18
-38
No files matched your search
+16
-27
@@ -1,19 +1,17 @@
|
||||
"""Target issue family: a target rejected day after day while the others are seen ("Degraded target"). visibility.py
|
||||
compares each sensor with its own usual level, so a target almost always rejected looks normal to it.
|
||||
"""Target issue family: a target rejected day after day ("Degraded target"); visibility.py compares each sensor with its
|
||||
own usual level, so a target almost always rejected looks normal to it. For a few days only, Target obscured
|
||||
(Obstruction family).
|
||||
|
||||
1. Calm instant for a sensor: at most CALM of the other sensors rejected (weather and fog, which touch everyone, are set
|
||||
aside).
|
||||
2. Degraded day: at least SHARE of the sensor's calm instants of the UTC day rejected; a day with fewer than MIN_HOURS of
|
||||
calm instants (at the station's cadence) does not count.
|
||||
3. Problem target: degraded days from a first to a last at least WEEK days apart, at most BRIDGE non-degraded days in a
|
||||
row between them, and SHARE of rejections over these days; the episode runs from its first rejected measurement to
|
||||
its last, greyed. The cause (dirt, vegetation, setting) is only seen on the images.
|
||||
1. Degraded day: at least SHARE of the sensor's measurements of the UTC day rejected; a day with fewer than MIN_HOURS of
|
||||
measurements (at the station's cadence) does not count.
|
||||
2. Run: degraded days with at most BRIDGE other days in a row between them, and SHARE of rejections over the run. WEEK
|
||||
days or more: Degraded target; MIN_DAYS or more: Target obscured. The episode runs from the first rejected
|
||||
measurement of the run to the last, greyed. The cause (dirt, vegetation, a parked vehicle) is seen on the images.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
from common import cadence, episode
|
||||
|
||||
CALM = 0.25 # share of the other sensors rejected, at most, at a calm instant
|
||||
SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %)
|
||||
MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants)
|
||||
BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day)
|
||||
@@ -26,14 +24,14 @@ def rejected(m):
|
||||
|
||||
|
||||
def _runs(days, min_instants):
|
||||
"""{day: [rejected, calm, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two
|
||||
"""{day: [rejected, measurements, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two
|
||||
degraded days belong to the run; a day that does not count is skipped."""
|
||||
runs, cur, gap = [], [], []
|
||||
for d in sorted(days):
|
||||
rej, calm = days[d][:2]
|
||||
if calm < min_instants:
|
||||
rej, n = days[d][:2]
|
||||
if n < min_instants:
|
||||
continue
|
||||
if rej / calm >= SHARE:
|
||||
if rej / n >= SHARE:
|
||||
cur += gap + [days[d]]
|
||||
gap = []
|
||||
elif cur:
|
||||
@@ -46,19 +44,11 @@ def _runs(days, min_instants):
|
||||
|
||||
def detect(measurements):
|
||||
min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements)
|
||||
by_t = {}
|
||||
days = {} # sensor -> day -> [rejected, measurements, first, last]
|
||||
for s, ms in measurements.items():
|
||||
for m in ms:
|
||||
by_t.setdefault(m["t"], {})[s] = rejected(m)
|
||||
days = {} # sensor -> day -> [rejected, calm, first measurement, last]
|
||||
for t in sorted(by_t):
|
||||
for s, r in by_t[t].items():
|
||||
others = [v for k, v in by_t[t].items() if k != s]
|
||||
c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t])
|
||||
c[3] = t
|
||||
if others and sum(others) / len(others) <= CALM:
|
||||
c[0] += r
|
||||
c[1] += 1
|
||||
c = days.setdefault(s, {}).setdefault(m["t"].date(), [0, 0, m["t"], m["t"]])
|
||||
c[0], c[1], c[3] = c[0] + rejected(m), c[1] + 1, m["t"]
|
||||
out = []
|
||||
for s, ds in sorted(days.items()):
|
||||
for run in _runs(ds, min_instants):
|
||||
@@ -67,8 +57,7 @@ def detect(measurements):
|
||||
continue
|
||||
bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)]
|
||||
n = sum(c[0] / c[1] >= SHARE for c in run)
|
||||
seen = f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were seen, on " \
|
||||
f"{n} of {len(run)} days."
|
||||
seen = f"{s} was rejected in {round(100 * share)} % of the measurements, on {n} of {len(run)} days."
|
||||
out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period "
|
||||
"and look at the images of the target.", [s], True) if len(run) >= WEEK else
|
||||
episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few "
|
||||
|
||||
+2
-11
@@ -132,12 +132,9 @@ def detect(measurements):
|
||||
# 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive
|
||||
# measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values).
|
||||
# 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO.
|
||||
# 3. Turbulent hours at most 2 h apart make an episode, dropped if more than REJECT_MAX of its measurements are rejected
|
||||
# beyond each sensor's habit (its share of rejections over the export): that is poor visibility. Greyed if the median
|
||||
# agitation reaches STRONG.
|
||||
# 3. Turbulent hours at most 2 h apart make an episode, greyed if the median agitation reaches STRONG.
|
||||
RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2)
|
||||
SHARE = 0.5
|
||||
REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; beyond the habit: wf-improve Visibility, mod. 1, S1
|
||||
STRONG = 2.0 # E-003 (×2.3 to ×2.6)
|
||||
HOUR = timedelta(hours=1)
|
||||
SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement."
|
||||
@@ -164,16 +161,10 @@ def heat_shimmer(measurements):
|
||||
hours.setdefault(h, {})[s] = (ag[h], er[h])
|
||||
turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
|
||||
and statistics.median(e for _, e in r.values()) >= RATIO]
|
||||
habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()}
|
||||
out = []
|
||||
for b in runs(turbulent, 2 * HOUR):
|
||||
start, end = b[0], b[-1] + HOUR
|
||||
by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()}
|
||||
new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items())
|
||||
if new > REJECT_MAX * sum(map(len, by.values())):
|
||||
continue
|
||||
agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}}
|
||||
strength = statistics.median(agit.values())
|
||||
out.append(episode(start, end, "Visibility", "Heat shimmer", SHIMMER.format(strength),
|
||||
out.append(episode(b[0], b[-1] + HOUR, "Visibility", "Heat shimmer", SHIMMER.format(strength),
|
||||
[s for s, v in agit.items() if v >= RATIO], strength >= STRONG))
|
||||
return out
|
||||
Reference in new issue
Block a user