REJECT_MAX of heat shimmer and CALM of target_issue removed
Heat shimmer is no longer dropped for its rejections beyond each sensor's habit (Visibility found +2: WILDSTRUBEL +7, NIEDERHORN +4). target_issue counts every measurement of the day, not only the calm instants (Obstruction found +3, correct -2: correct -11 at LENZSPITZE and -12 at WILDSTRUBEL; Target issue unchanged). Benchmark Totals (from the previous commit): Visibility found 55 -> 57 %; Obstruction found 42 -> 45 %, correct 33 -> 31 %; the other families unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4188efe201
commit
ab0347c41d
2 files changed
+18
-38
No files matched your search
+16
-27
@@ -1,19 +1,17 @@
|
|||||||
"""Target issue family: a target rejected day after day while the others are seen ("Degraded target"). visibility.py
|
"""Target issue family: a target rejected day after day ("Degraded target"); visibility.py compares each sensor with its
|
||||||
compares each sensor with its own usual level, so a target almost always rejected looks normal to it.
|
own usual level, so a target almost always rejected looks normal to it. For a few days only, Target obscured
|
||||||
|
(Obstruction family).
|
||||||
|
|
||||||
1. Calm instant for a sensor: at most CALM of the other sensors rejected (weather and fog, which touch everyone, are set
|
1. Degraded day: at least SHARE of the sensor's measurements of the UTC day rejected; a day with fewer than MIN_HOURS of
|
||||||
aside).
|
measurements (at the station's cadence) does not count.
|
||||||
2. Degraded day: at least SHARE of the sensor's calm instants of the UTC day rejected; a day with fewer than MIN_HOURS of
|
2. Run: degraded days with at most BRIDGE other days in a row between them, and SHARE of rejections over the run. WEEK
|
||||||
calm instants (at the station's cadence) does not count.
|
days or more: Degraded target; MIN_DAYS or more: Target obscured. The episode runs from the first rejected
|
||||||
3. Problem target: degraded days from a first to a last at least WEEK days apart, at most BRIDGE non-degraded days in a
|
measurement of the run to the last, greyed. The cause (dirt, vegetation, a parked vehicle) is seen on the images.
|
||||||
row between them, and SHARE of rejections over these days; the episode runs from its first rejected measurement to
|
|
||||||
its last, greyed. The cause (dirt, vegetation, setting) is only seen on the images.
|
|
||||||
"""
|
"""
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
from common import cadence, episode
|
from common import cadence, episode
|
||||||
|
|
||||||
CALM = 0.25 # share of the other sensors rejected, at most, at a calm instant
|
|
||||||
SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %)
|
SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %)
|
||||||
MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants)
|
MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants)
|
||||||
BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day)
|
BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day)
|
||||||
@@ -26,14 +24,14 @@ def rejected(m):
|
|||||||
|
|
||||||
|
|
||||||
def _runs(days, min_instants):
|
def _runs(days, min_instants):
|
||||||
"""{day: [rejected, calm, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two
|
"""{day: [rejected, measurements, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two
|
||||||
degraded days belong to the run; a day that does not count is skipped."""
|
degraded days belong to the run; a day that does not count is skipped."""
|
||||||
runs, cur, gap = [], [], []
|
runs, cur, gap = [], [], []
|
||||||
for d in sorted(days):
|
for d in sorted(days):
|
||||||
rej, calm = days[d][:2]
|
rej, n = days[d][:2]
|
||||||
if calm < min_instants:
|
if n < min_instants:
|
||||||
continue
|
continue
|
||||||
if rej / calm >= SHARE:
|
if rej / n >= SHARE:
|
||||||
cur += gap + [days[d]]
|
cur += gap + [days[d]]
|
||||||
gap = []
|
gap = []
|
||||||
elif cur:
|
elif cur:
|
||||||
@@ -46,19 +44,11 @@ def _runs(days, min_instants):
|
|||||||
|
|
||||||
def detect(measurements):
|
def detect(measurements):
|
||||||
min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements)
|
min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements)
|
||||||
by_t = {}
|
days = {} # sensor -> day -> [rejected, measurements, first, last]
|
||||||
for s, ms in measurements.items():
|
for s, ms in measurements.items():
|
||||||
for m in ms:
|
for m in ms:
|
||||||
by_t.setdefault(m["t"], {})[s] = rejected(m)
|
c = days.setdefault(s, {}).setdefault(m["t"].date(), [0, 0, m["t"], m["t"]])
|
||||||
days = {} # sensor -> day -> [rejected, calm, first measurement, last]
|
c[0], c[1], c[3] = c[0] + rejected(m), c[1] + 1, m["t"]
|
||||||
for t in sorted(by_t):
|
|
||||||
for s, r in by_t[t].items():
|
|
||||||
others = [v for k, v in by_t[t].items() if k != s]
|
|
||||||
c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t])
|
|
||||||
c[3] = t
|
|
||||||
if others and sum(others) / len(others) <= CALM:
|
|
||||||
c[0] += r
|
|
||||||
c[1] += 1
|
|
||||||
out = []
|
out = []
|
||||||
for s, ds in sorted(days.items()):
|
for s, ds in sorted(days.items()):
|
||||||
for run in _runs(ds, min_instants):
|
for run in _runs(ds, min_instants):
|
||||||
@@ -67,8 +57,7 @@ def detect(measurements):
|
|||||||
continue
|
continue
|
||||||
bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)]
|
bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)]
|
||||||
n = sum(c[0] / c[1] >= SHARE for c in run)
|
n = sum(c[0] / c[1] >= SHARE for c in run)
|
||||||
seen = f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were seen, on " \
|
seen = f"{s} was rejected in {round(100 * share)} % of the measurements, on {n} of {len(run)} days."
|
||||||
f"{n} of {len(run)} days."
|
|
||||||
out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period "
|
out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period "
|
||||||
"and look at the images of the target.", [s], True) if len(run) >= WEEK else
|
"and look at the images of the target.", [s], True) if len(run) >= WEEK else
|
||||||
episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few "
|
episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few "
|
||||||
|
|||||||
+2
-11
@@ -132,12 +132,9 @@ def detect(measurements):
|
|||||||
# 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive
|
# 1. Per sensor and per hour H, over [H - 1 h, H + 2 h): agitation, the median of |Δcorr_x| + |Δcorr_y| between successive
|
||||||
# measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values).
|
# measurements, and the median error_value; each divided by its calm (lower quartile of the sensor's hourly values).
|
||||||
# 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO.
|
# 2. Turbulent hour: at least SHARE of the sensors agitated at RATIO times their calm, and median error_value at RATIO.
|
||||||
# 3. Turbulent hours at most 2 h apart make an episode, dropped if more than REJECT_MAX of its measurements are rejected
|
# 3. Turbulent hours at most 2 h apart make an episode, greyed if the median agitation reaches STRONG.
|
||||||
# beyond each sensor's habit (its share of rejections over the export): that is poor visibility. Greyed if the median
|
|
||||||
# agitation reaches STRONG.
|
|
||||||
RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2)
|
RATIO = 1.5 # E-003 (WILDSTRUBEL: by day, corr ×1.7 to ×3 the night, error_value ×1.7 to ×3.2)
|
||||||
SHARE = 0.5
|
SHARE = 0.5
|
||||||
REJECT_MAX = 0.15 # E-003 ≤ 7 %; cobweb (E-002) and rain (E-005) ≥ 18 %; beyond the habit: wf-improve Visibility, mod. 1, S1
|
|
||||||
STRONG = 2.0 # E-003 (×2.3 to ×2.6)
|
STRONG = 2.0 # E-003 (×2.3 to ×2.6)
|
||||||
HOUR = timedelta(hours=1)
|
HOUR = timedelta(hours=1)
|
||||||
SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement."
|
SHIMMER = "Warm air made the readings shake ({:.1f}× noisier than in calm hours). The small fast wiggles are not movement."
|
||||||
@@ -164,16 +161,10 @@ def heat_shimmer(measurements):
|
|||||||
hours.setdefault(h, {})[s] = (ag[h], er[h])
|
hours.setdefault(h, {})[s] = (ag[h], er[h])
|
||||||
turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
|
turbulent = [h for h, r in sorted(hours.items()) if sum(a >= RATIO for a, _ in r.values()) >= SHARE * len(r)
|
||||||
and statistics.median(e for _, e in r.values()) >= RATIO]
|
and statistics.median(e for _, e in r.values()) >= RATIO]
|
||||||
habit = {s: sum(m["corr_x"] is None for m in ms) / len(ms) for s, ms in measurements.items()}
|
|
||||||
out = []
|
out = []
|
||||||
for b in runs(turbulent, 2 * HOUR):
|
for b in runs(turbulent, 2 * HOUR):
|
||||||
start, end = b[0], b[-1] + HOUR
|
|
||||||
by = {s: [m for m in ms if start <= m["t"] < end] for s, ms in measurements.items()}
|
|
||||||
new = sum(max(0, sum(m["corr_x"] is None for m in d) - habit[s] * len(d)) for s, d in by.items())
|
|
||||||
if new > REJECT_MAX * sum(map(len, by.values())):
|
|
||||||
continue
|
|
||||||
agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}}
|
agit = {s: statistics.median(hours[h][s][0] for h in b if s in hours[h]) for s in {s for h in b for s in hours[h]}}
|
||||||
strength = statistics.median(agit.values())
|
strength = statistics.median(agit.values())
|
||||||
out.append(episode(start, end, "Visibility", "Heat shimmer", SHIMMER.format(strength),
|
out.append(episode(b[0], b[-1] + HOUR, "Visibility", "Heat shimmer", SHIMMER.format(strength),
|
||||||
[s for s, v in agit.items() if v >= RATIO], strength >= STRONG))
|
[s for s, v in agit.items() if v >= RATIO], strength >= STRONG))
|
||||||
return out
|
return out
|
||||||
Reference in new issue
Block a user