From ec4b900fa0e9258188057dfc820c18e9e9850913 Mon Sep 17 00:00:00 2001 From: maxime Date: Mon, 16 Nov 2026 16:38:05 +0100 Subject: [PATCH] target_issue: one rule of degraded days; Target stuck removed A single bridged run of degraded days (SHARE of the calm instants rejected, BRIDGE): WEEK days or more, Degraded target; MIN_DAYS or more, Target obscured (Obstruction), from its first rejected measurement to its last. Removed: the separate unbridged runs at half of the calm instants (old SHARE) and their episodes per run of rejections, Target stuck and STUCK (no instant of the benchmark), the cadence helper (common.cadence). Benchmark Totals (from the previous commit): Obstruction found 22 -> 42 %, correct 32 -> 33 %; the other families unchanged. Co-Authored-By: Claude Opus 5.5 --- scripts/target_issue.py | 197 ++++++++++------------------------------ 1 file changed, 50 insertions(+), 147 deletions(-) diff --git a/scripts/target_issue.py b/scripts/target_issue.py index 97799ab..1132197 100644 --- a/scripts/target_issue.py +++ b/scripts/target_issue.py @@ -1,78 +1,59 @@ -"""Detector of the Target issue family (the families document): a persistently problematic target: rejected most of the -time, day after day, while the others are seen (degraded target), stuck (end of file), or flipping (the jumps of -displacement.py, "Target flipping"). A target rejected for less than a week is hidden at times (point 4): episodes() -returns it with the Obstruction family, which detect() leaves to obstruction.py. +"""Target issue family: a target rejected day after day while the others are seen ("Degraded target"). visibility.py +compares each sensor with its own usual level, so a target almost always rejected looks normal to it. -visibility.py compares each sensor with its own usual level: a target that is almost always rejected looks normal to it -(FLORIDA, October 2026: Massif 2 and Sellette 6 rejected half the measurements or more, every day). -Method: -1. Calm instant for a sensor: at most CALM of the other sensors rejected (error_value > error_threshold); weather and - fog, which touch everyone, are thus set aside. -2. Per UTC day and per sensor, on its calm instants (at least MIN_HOURS hours of instants at the station's cadence, - median gap between two measurement instants; otherwise the day does not count): degraded day if at least SHARE of - them are rejected. Cadence (wf-improve NEBRASKA, batch 01): 12 fixed instants, "3 h at 15 min", counted no day at one - measurement every 4 h; REFCOLH22, rejected all the time since it was lost, was not seen. -3. Episode: at least MIN_DAYS degraded days in a row (a day that does not count does not break the run), from the first - rejected measurement of these days to the last; ignore = True: the target's curve is not reliable. -4. Hindered all the time or at times (boundary "Target issue or Obstruction" of the families document; wf-improve - NEBRASKA, batch 01): a run of at least WEEK days is a problem target, "Degraded target"; shorter, a target hidden at - times, "Target obscured" (Obstruction family), one episode per run of rejected measurements (the hours when it is - seen are not part of it). BREITHORN: vehicles parked in front of COLI24 and POUI22, 2 to 5 days; NEBRASKA and - MICHIGAN: a lost reference, 25 days; JUNGFRAU: PM_G, 63. - For the problem target, the run is bridged: at most BRIDGE non-degraded days in a row do not break it, and it then - needs WEEK days from its first to its last degraded day, at SHARE_WEEK of rejections over all its days ("rejections - specific to the target almost every day of the range"); the Target obscured episodes it covers no longer exist - (wf-improve session Obstruction, mod. 1, S1: FLORIDA Sellette 6 and Massif 2, SNCF01 F4.3 and F4.4, NORDEND CTR01, - NIEDERHORN e01, cut into Target obscured by a day under SHARE; BREITHORN, parked vehicles, unchanged). - The problem target judges regularity rather than the share of the day: a day is degraded from SHARE_WEEK (a quarter) - of its calm instants rejected, and BRIDGE is one day; the target hidden at times keeps SHARE. A target rejected every - day at the same hours, by day only or by night only, stays under half and was not seen (wf-improve session Target - issue, mod. 1, S1: WILDSTRUBEL R3, 04:00 to 16:00 every day; NORDEND PR04 and CTR01, NIEDERHORN PWD_IMP_B08_024_C09 - and e01, SNCF01 F4.3). -The cause (dirt, vegetation, setting) can only be read on the images. -Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per UTC day, the noise of its corr_x and -corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on -both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at -7e-5 px and C3 at 2e-4 against 0.013 for the others. +1. Calm instant for a sensor: at most CALM of the other sensors rejected (weather and fog, which touch everyone, are set + aside). +2. Degraded day: at least SHARE of the sensor's calm instants of the UTC day rejected; a day with fewer than MIN_HOURS of + calm instants (at the station's cadence) does not count. +3. Problem target: degraded days from a first to a last at least WEEK days apart, at most BRIDGE non-degraded days in a + row between them, and SHARE of rejections over these days; the episode runs from its first rejected measurement to + its last, greyed. The cause (dirt, vegetation, setting) is only seen on the images. """ -import statistics +from datetime import timedelta -from data import is_reference, iso +from common import cadence, episode -CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant -SHARE = 0.5 # half of the calm instants rejected: "most of the time" -SHARE_WEEK = 0.25 # a quarter of the calm instants rejected, each day of a week or more: a problem target, even when - # rejected by day only or by night only (wf-improve session Target issue, mod. 1, S1) -MIN_HOURS = 3 # hours of calm instants, at the station's cadence: below that, the day does not count -MIN_DAYS = 2 # a single day is too often an accident (NORDEND: isolated days on 16 sensors) -BRIDGE = 1 # non-degraded days that do not break a run of degraded days (4); at 2, the parked vehicles of BREITHORN - # (two days, then nothing for two days, then one) would make a week at a quarter -WEEK = 7 # degraded days in a row for a problem target; below, a target hidden at times -STUCK = 0.05 # noise of a stuck target compared with the others (E-011: 0.006; NORDEND, E-001: 0.7 at the lowest) +CALM = 0.25 # share of the other sensors rejected, at most, at a calm instant +SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %) +MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants) +BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day) +WEEK = 7 # wf-improve NEBRASKA, batch 01 (lost references, 25 days; parked vehicles, 2 to 5 days, are not one) +MIN_DAYS = 2 # shorter than WEEK: a target hidden for a few days, Target obscured (Obstruction); NORDEND: isolated days -def is_rejected(m): +def rejected(m): return m["error_value"] is not None and m["error_threshold"] is not None and m["error_value"] > m["error_threshold"] -def cadence(measurements): - """Median gap between two measurement instants, all sensors together, in hours (as no_data).""" - ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms}) - return statistics.median((b - a).total_seconds() for a, b in zip(ts, ts[1:])) / 3600 if len(ts) > 1 else 1 +def _runs(days, min_instants): + """{day: [rejected, calm, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two + degraded days belong to the run; a day that does not count is skipped.""" + runs, cur, gap = [], [], [] + for d in sorted(days): + rej, calm = days[d][:2] + if calm < min_instants: + continue + if rej / calm >= SHARE: + cur += gap + [days[d]] + gap = [] + elif cur: + gap.append(days[d]) + if len(gap) > BRIDGE: + runs.append(cur) + cur, gap = [], [] + return runs + ([cur] if cur else []) -def episodes(measurements): - """Targets rejected day after day: Target issue (a week or more) and Obstruction (less), then the stuck ones.""" - min_instants = MIN_HOURS / cadence(measurements) +def detect(measurements): + min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements) by_t = {} for s, ms in measurements.items(): for m in ms: - by_t.setdefault(m["t"], {})[s] = is_rejected(m) + by_t.setdefault(m["t"], {})[s] = rejected(m) days = {} # sensor -> day -> [rejected, calm, first measurement, last] for t in sorted(by_t): - d = by_t[t] - for s, r in d.items(): - others = [v for k, v in d.items() if k != s] + for s, r in by_t[t].items(): + others = [v for k, v in by_t[t].items() if k != s] c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t]) c[3] = t if others and sum(others) / len(others) <= CALM: @@ -80,94 +61,16 @@ def episodes(measurements): c[1] += 1 out = [] for s, ds in sorted(days.items()): - covered = [] # bridged runs of at least WEEK days, at SHARE_WEEK of rejections: a problem target - for run in _runs(ds, min_instants, BRIDGE, SHARE_WEEK): + for run in _runs(ds, min_instants): share = sum(c[0] for c in run) / sum(c[1] for c in run) - if len(run) < WEEK or share < SHARE_WEEK: + if len(run) < MIN_DAYS or share < SHARE: continue - n = sum(c[0] / c[1] >= SHARE_WEEK for c in run) - r = _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]]) - covered.append((run[0][2], run[-1][3])) - out.append({"start": iso(r[0][0]), "end": iso(r[-1][1]), "kind": "Target issue", "label": "Degraded target", - "text": f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were " - f"seen, on {n} of {len(run)} days.", "ignore": True, "sensors": [s]}) - for run in _runs(ds, min_instants, 0, SHARE): # at times: one episode per run of rejected measurements - if len(run) < MIN_DAYS or any(a <= run[0][2] <= b for a, b in covered): - continue - share = round(100 * sum(c[0] for c in run) / sum(c[1] for c in run)) - text = f"{s} was rejected in {share} % of the measurements when the other targets were seen, {len(run)} days in a row." - out += [{"start": iso(a), "end": iso(b), "kind": "Obstruction", "label": "Target obscured", "text": text, - "ignore": True, "sensors": [s]} - for a, b in _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])] - return sorted(out + stuck(measurements, min_instants), key=lambda e: e["start"]) - - -def _runs(ds, min_instants, bridge, share): - """{day: [rejected, calm, first, last]} -> runs of degraded days; at most `bridge` non-degraded days in a row between - two degraded days do not break the run (they belong to it); a day that does not count does not break it.""" - runs, cur, gap = [], [], [] - for j in sorted(ds): - rej, calm = ds[j][:2] - if calm < min_instants: - continue - if rej / calm >= share: - cur += gap + [ds[j]] - gap = [] - elif cur: - gap.append(ds[j]) - if len(gap) > bridge: - runs.append(cur) - cur, gap = [], [] - return runs + ([cur] if cur else []) - - -def _rejections(ms): - """Measurements of a sensor -> [(start, end)] of its runs of rejected measurements.""" - out, cur = [], [] - for m in ms + [None]: - if m and is_rejected(m): - cur.append(m["t"]) - elif cur: - out.append((cur[0], cur[-1])) - cur = [] + bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)] + n = sum(c[0] / c[1] >= SHARE for c in run) + seen = f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were seen, on " \ + f"{n} of {len(run)} days." + out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period " + "and look at the images of the target.", [s], True) if len(run) >= WEEK else + episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few " + "days: ignore its curve in this period.", [s], True)) return out - - -def _noise(v): - return statistics.median(abs(y - x) for x, y in zip(v, v[1:])) if len(v) > 2 else None - - -def stuck(measurements, min_instants): - """Targets whose corr_x and corr_y barely move, day after day, while the others move.""" - days = {} # day -> target -> (x noise, y noise, first measurement, last) - for s, ms in measurements.items(): - if is_reference(ms): - continue - by_day = {} - for m in ms: - by_day.setdefault(m["t"].date(), []).append(m) - for d, dm in by_day.items(): - b = [_noise([m[c] for m in dm if m[c] is not None]) for c in ("corr_x", "corr_y")] - if None not in b and len(dm) >= min_instants: - days.setdefault(d, {})[s] = (*b, dm[0]["t"], dm[-1]["t"]) - runs = {} # target -> [[stuck days in a row]] - for d in sorted(days): - cs = days[d] - for s, (bx, by, a, b) in cs.items(): - others = [v for k, (x, y, *_) in cs.items() if k != s for v in (x, y)] - if len(others) < 4: - continue - ref = statistics.median(others) - ru = runs.setdefault(s, [[]]) - if ref and max(bx, by) <= STUCK * ref: - ru[-1].append((a, b)) - elif ru[-1]: - ru.append([]) - return [{"start": iso(k[0][0]), "end": iso(k[-1][1]), "kind": "Target issue", "label": "Target stuck", - "text": f"{s} barely moved while the other targets did ({len(k)} day{'s' if len(k) > 1 else ''}): it may be " - "set on a reference prism.", "ignore": True, "sensors": [s]} - for s, ru in sorted(runs.items()) for k in ru if k] - - -def detect(measurements): - return episodes(measurements)