diff --git a/scripts/target_issue.py b/scripts/target_issue.py index 97799ab..1132197 100644 --- a/scripts/target_issue.py +++ b/scripts/target_issue.py @@ -1,78 +1,59 @@ -"""Detector of the Target issue family (the families document): a persistently problematic target: rejected most of the -time, day after day, while the others are seen (degraded target), stuck (end of file), or flipping (the jumps of -displacement.py, "Target flipping"). A target rejected for less than a week is hidden at times (point 4): episodes() -returns it with the Obstruction family, which detect() leaves to obstruction.py. +"""Target issue family: a target rejected day after day while the others are seen ("Degraded target"). visibility.py +compares each sensor with its own usual level, so a target almost always rejected looks normal to it. -visibility.py compares each sensor with its own usual level: a target that is almost always rejected looks normal to it -(FLORIDA, October 2026: Massif 2 and Sellette 6 rejected half the measurements or more, every day). -Method: -1. Calm instant for a sensor: at most CALM of the other sensors rejected (error_value > error_threshold); weather and - fog, which touch everyone, are thus set aside. -2. Per UTC day and per sensor, on its calm instants (at least MIN_HOURS hours of instants at the station's cadence, - median gap between two measurement instants; otherwise the day does not count): degraded day if at least SHARE of - them are rejected. Cadence (wf-improve NEBRASKA, batch 01): 12 fixed instants, "3 h at 15 min", counted no day at one - measurement every 4 h; REFCOLH22, rejected all the time since it was lost, was not seen. -3. Episode: at least MIN_DAYS degraded days in a row (a day that does not count does not break the run), from the first - rejected measurement of these days to the last; ignore = True: the target's curve is not reliable. -4. Hindered all the time or at times (boundary "Target issue or Obstruction" of the families document; wf-improve - NEBRASKA, batch 01): a run of at least WEEK days is a problem target, "Degraded target"; shorter, a target hidden at - times, "Target obscured" (Obstruction family), one episode per run of rejected measurements (the hours when it is - seen are not part of it). BREITHORN: vehicles parked in front of COLI24 and POUI22, 2 to 5 days; NEBRASKA and - MICHIGAN: a lost reference, 25 days; JUNGFRAU: PM_G, 63. - For the problem target, the run is bridged: at most BRIDGE non-degraded days in a row do not break it, and it then - needs WEEK days from its first to its last degraded day, at SHARE_WEEK of rejections over all its days ("rejections - specific to the target almost every day of the range"); the Target obscured episodes it covers no longer exist - (wf-improve session Obstruction, mod. 1, S1: FLORIDA Sellette 6 and Massif 2, SNCF01 F4.3 and F4.4, NORDEND CTR01, - NIEDERHORN e01, cut into Target obscured by a day under SHARE; BREITHORN, parked vehicles, unchanged). - The problem target judges regularity rather than the share of the day: a day is degraded from SHARE_WEEK (a quarter) - of its calm instants rejected, and BRIDGE is one day; the target hidden at times keeps SHARE. A target rejected every - day at the same hours, by day only or by night only, stays under half and was not seen (wf-improve session Target - issue, mod. 1, S1: WILDSTRUBEL R3, 04:00 to 16:00 every day; NORDEND PR04 and CTR01, NIEDERHORN PWD_IMP_B08_024_C09 - and e01, SNCF01 F4.3). -The cause (dirt, vegetation, setting) can only be read on the images. -Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per UTC day, the noise of its corr_x and -corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on -both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at -7e-5 px and C3 at 2e-4 against 0.013 for the others. +1. Calm instant for a sensor: at most CALM of the other sensors rejected (weather and fog, which touch everyone, are set + aside). +2. Degraded day: at least SHARE of the sensor's calm instants of the UTC day rejected; a day with fewer than MIN_HOURS of + calm instants (at the station's cadence) does not count. +3. Problem target: degraded days from a first to a last at least WEEK days apart, at most BRIDGE non-degraded days in a + row between them, and SHARE of rejections over these days; the episode runs from its first rejected measurement to + its last, greyed. The cause (dirt, vegetation, setting) is only seen on the images. """ -import statistics +from datetime import timedelta -from data import is_reference, iso +from common import cadence, episode -CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant -SHARE = 0.5 # half of the calm instants rejected: "most of the time" -SHARE_WEEK = 0.25 # a quarter of the calm instants rejected, each day of a week or more: a problem target, even when - # rejected by day only or by night only (wf-improve session Target issue, mod. 1, S1) -MIN_HOURS = 3 # hours of calm instants, at the station's cadence: below that, the day does not count -MIN_DAYS = 2 # a single day is too often an accident (NORDEND: isolated days on 16 sensors) -BRIDGE = 1 # non-degraded days that do not break a run of degraded days (4); at 2, the parked vehicles of BREITHORN - # (two days, then nothing for two days, then one) would make a week at a quarter -WEEK = 7 # degraded days in a row for a problem target; below, a target hidden at times -STUCK = 0.05 # noise of a stuck target compared with the others (E-011: 0.006; NORDEND, E-001: 0.7 at the lowest) +CALM = 0.25 # share of the other sensors rejected, at most, at a calm instant +SHARE = 0.25 # wf-improve Target issue, mod. 1, S1 (WILDSTRUBEL R3 rejected from 04:00 to 16:00 every day, 30 to 45 %) +MIN_HOURS = 3 # wf-improve NEBRASKA, batch 01 (one measurement every 4 h: as hours, not as instants) +BRIDGE = 1 # wf-improve Obstruction, mod. 1, S1 and Target issue, mod. 1, S1 (problem targets cut by a better day) +WEEK = 7 # wf-improve NEBRASKA, batch 01 (lost references, 25 days; parked vehicles, 2 to 5 days, are not one) +MIN_DAYS = 2 # shorter than WEEK: a target hidden for a few days, Target obscured (Obstruction); NORDEND: isolated days -def is_rejected(m): +def rejected(m): return m["error_value"] is not None and m["error_threshold"] is not None and m["error_value"] > m["error_threshold"] -def cadence(measurements): - """Median gap between two measurement instants, all sensors together, in hours (as no_data).""" - ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms}) - return statistics.median((b - a).total_seconds() for a, b in zip(ts, ts[1:])) / 3600 if len(ts) > 1 else 1 +def _runs(days, min_instants): + """{day: [rejected, calm, first, last]} -> runs of degraded days; at most BRIDGE other days in a row between two + degraded days belong to the run; a day that does not count is skipped.""" + runs, cur, gap = [], [], [] + for d in sorted(days): + rej, calm = days[d][:2] + if calm < min_instants: + continue + if rej / calm >= SHARE: + cur += gap + [days[d]] + gap = [] + elif cur: + gap.append(days[d]) + if len(gap) > BRIDGE: + runs.append(cur) + cur, gap = [], [] + return runs + ([cur] if cur else []) -def episodes(measurements): - """Targets rejected day after day: Target issue (a week or more) and Obstruction (less), then the stuck ones.""" - min_instants = MIN_HOURS / cadence(measurements) +def detect(measurements): + min_instants = timedelta(hours=MIN_HOURS) / cadence(measurements) by_t = {} for s, ms in measurements.items(): for m in ms: - by_t.setdefault(m["t"], {})[s] = is_rejected(m) + by_t.setdefault(m["t"], {})[s] = rejected(m) days = {} # sensor -> day -> [rejected, calm, first measurement, last] for t in sorted(by_t): - d = by_t[t] - for s, r in d.items(): - others = [v for k, v in d.items() if k != s] + for s, r in by_t[t].items(): + others = [v for k, v in by_t[t].items() if k != s] c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t]) c[3] = t if others and sum(others) / len(others) <= CALM: @@ -80,94 +61,16 @@ def episodes(measurements): c[1] += 1 out = [] for s, ds in sorted(days.items()): - covered = [] # bridged runs of at least WEEK days, at SHARE_WEEK of rejections: a problem target - for run in _runs(ds, min_instants, BRIDGE, SHARE_WEEK): + for run in _runs(ds, min_instants): share = sum(c[0] for c in run) / sum(c[1] for c in run) - if len(run) < WEEK or share < SHARE_WEEK: + if len(run) < MIN_DAYS or share < SHARE: continue - n = sum(c[0] / c[1] >= SHARE_WEEK for c in run) - r = _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]]) - covered.append((run[0][2], run[-1][3])) - out.append({"start": iso(r[0][0]), "end": iso(r[-1][1]), "kind": "Target issue", "label": "Degraded target", - "text": f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were " - f"seen, on {n} of {len(run)} days.", "ignore": True, "sensors": [s]}) - for run in _runs(ds, min_instants, 0, SHARE): # at times: one episode per run of rejected measurements - if len(run) < MIN_DAYS or any(a <= run[0][2] <= b for a, b in covered): - continue - share = round(100 * sum(c[0] for c in run) / sum(c[1] for c in run)) - text = f"{s} was rejected in {share} % of the measurements when the other targets were seen, {len(run)} days in a row." - out += [{"start": iso(a), "end": iso(b), "kind": "Obstruction", "label": "Target obscured", "text": text, - "ignore": True, "sensors": [s]} - for a, b in _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])] - return sorted(out + stuck(measurements, min_instants), key=lambda e: e["start"]) - - -def _runs(ds, min_instants, bridge, share): - """{day: [rejected, calm, first, last]} -> runs of degraded days; at most `bridge` non-degraded days in a row between - two degraded days do not break the run (they belong to it); a day that does not count does not break it.""" - runs, cur, gap = [], [], [] - for j in sorted(ds): - rej, calm = ds[j][:2] - if calm < min_instants: - continue - if rej / calm >= share: - cur += gap + [ds[j]] - gap = [] - elif cur: - gap.append(ds[j]) - if len(gap) > bridge: - runs.append(cur) - cur, gap = [], [] - return runs + ([cur] if cur else []) - - -def _rejections(ms): - """Measurements of a sensor -> [(start, end)] of its runs of rejected measurements.""" - out, cur = [], [] - for m in ms + [None]: - if m and is_rejected(m): - cur.append(m["t"]) - elif cur: - out.append((cur[0], cur[-1])) - cur = [] + bad = [m["t"] for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3] and rejected(m)] + n = sum(c[0] / c[1] >= SHARE for c in run) + seen = f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were seen, on " \ + f"{n} of {len(run)} days." + out.append(episode(bad[0], bad[-1], "Target issue", "Degraded target", seen + " Ignore its curve in this period " + "and look at the images of the target.", [s], True) if len(run) >= WEEK else + episode(bad[0], bad[-1], "Obstruction", "Target obscured", seen + " Something hid it for a few " + "days: ignore its curve in this period.", [s], True)) return out - - -def _noise(v): - return statistics.median(abs(y - x) for x, y in zip(v, v[1:])) if len(v) > 2 else None - - -def stuck(measurements, min_instants): - """Targets whose corr_x and corr_y barely move, day after day, while the others move.""" - days = {} # day -> target -> (x noise, y noise, first measurement, last) - for s, ms in measurements.items(): - if is_reference(ms): - continue - by_day = {} - for m in ms: - by_day.setdefault(m["t"].date(), []).append(m) - for d, dm in by_day.items(): - b = [_noise([m[c] for m in dm if m[c] is not None]) for c in ("corr_x", "corr_y")] - if None not in b and len(dm) >= min_instants: - days.setdefault(d, {})[s] = (*b, dm[0]["t"], dm[-1]["t"]) - runs = {} # target -> [[stuck days in a row]] - for d in sorted(days): - cs = days[d] - for s, (bx, by, a, b) in cs.items(): - others = [v for k, (x, y, *_) in cs.items() if k != s for v in (x, y)] - if len(others) < 4: - continue - ref = statistics.median(others) - ru = runs.setdefault(s, [[]]) - if ref and max(bx, by) <= STUCK * ref: - ru[-1].append((a, b)) - elif ru[-1]: - ru.append([]) - return [{"start": iso(k[0][0]), "end": iso(k[-1][1]), "kind": "Target issue", "label": "Target stuck", - "text": f"{s} barely moved while the other targets did ({len(k)} day{'s' if len(k) > 1 else ''}): it may be " - "set on a reference prism.", "ignore": True, "sensors": [s]} - for s, ru in sorted(runs.items()) for k in ru if k] - - -def detect(measurements): - return episodes(measurements)