"""Detector of the Target issue family (the families document): a persistently problematic target: rejected most of the time, day after day, while the others are seen (degraded target), stuck (end of file), or flipping (the jumps of displacement.py, "Target flipping"). A target rejected for less than a week is hidden at times (point 4): episodes() returns it with the Obstruction family, which detect() leaves to obstruction.py. visibility.py compares each sensor with its own usual level: a target that is almost always rejected looks normal to it (FLORIDA, October 2026: Massif 2 and Sellette 6 rejected half the measurements or more, every day). Method: 1. Calm instant for a sensor: at most CALM of the other sensors rejected (error_value > error_threshold); weather and fog, which touch everyone, are thus set aside. 2. Per UTC day and per sensor, on its calm instants (at least MIN_HOURS hours of instants at the station's cadence, median gap between two measurement instants; otherwise the day does not count): degraded day if at least SHARE of them are rejected. Cadence (wf-improve NEBRASKA, batch 01): 12 fixed instants, "3 h at 15 min", counted no day at one measurement every 4 h; REFCOLH22, rejected all the time since it was lost, was not seen. 3. Episode: at least MIN_DAYS degraded days in a row (a day that does not count does not break the run), from the first rejected measurement of these days to the last; ignore = True: the target's curve is not reliable. 4. Hindered all the time or at times (boundary "Target issue or Obstruction" of the families document; wf-improve NEBRASKA, batch 01): a run of at least WEEK days is a problem target, "Degraded target"; shorter, a target hidden at times, "Target obscured" (Obstruction family), one episode per run of rejected measurements (the hours when it is seen are not part of it). BREITHORN: vehicles parked in front of COLI24 and POUI22, 2 to 5 days; NEBRASKA and MICHIGAN: a lost reference, 25 days; JUNGFRAU: PM_G, 63. For the problem target, the run is bridged: at most BRIDGE non-degraded days in a row do not break it, and it then needs WEEK days from its first to its last degraded day, at SHARE_WEEK of rejections over all its days ("rejections specific to the target almost every day of the range"); the Target obscured episodes it covers no longer exist (wf-improve session Obstruction, mod. 1, S1: FLORIDA Sellette 6 and Massif 2, SNCF01 F4.3 and F4.4, NORDEND CTR01, NIEDERHORN e01, cut into Target obscured by a day under SHARE; BREITHORN, parked vehicles, unchanged). The problem target judges regularity rather than the share of the day: a day is degraded from SHARE_WEEK (a quarter) of its calm instants rejected, and BRIDGE is one day; the target hidden at times keeps SHARE. A target rejected every day at the same hours, by day only or by night only, stays under half and was not seen (wf-improve session Target issue, mod. 1, S1: WILDSTRUBEL R3, 04:00 to 16:00 every day; NORDEND PR04 and CTR01, NIEDERHORN PWD_IMP_B08_024_C09 and e01, SNCF01 F4.3). The cause (dirt, vegetation, setting) can only be read on the images. Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per UTC day, the noise of its corr_x and corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at 7e-5 px and C3 at 2e-4 against 0.013 for the others. Usage: python3 detectors/target_issue.py (prints the episodes) """ import os import statistics import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it import displacement # noqa: E402 from data import is_reference, iso, load # noqa: E402 CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant SHARE = 0.5 # half of the calm instants rejected: "most of the time" SHARE_WEEK = 0.25 # a quarter of the calm instants rejected, each day of a week or more: a problem target, even when # rejected by day only or by night only (wf-improve session Target issue, mod. 1, S1) MIN_HOURS = 3 # hours of calm instants, at the station's cadence: below that, the day does not count MIN_DAYS = 2 # a single day is too often an accident (NORDEND: isolated days on 16 sensors) BRIDGE = 1 # non-degraded days that do not break a run of degraded days (4); at 2, the parked vehicles of BREITHORN # (two days, then nothing for two days, then one) would make a week at a quarter WEEK = 7 # degraded days in a row for a problem target; below, a target hidden at times STUCK = 0.05 # noise of a stuck target compared with the others (E-011: 0.006; NORDEND, E-001: 0.7 at the lowest) def is_rejected(m): return m["error_value"] is not None and m["error_threshold"] is not None and m["error_value"] > m["error_threshold"] def cadence(measurements): """Median gap between two measurement instants, all sensors together, in hours (as no_data).""" ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms}) return statistics.median((b - a).total_seconds() for a, b in zip(ts, ts[1:])) / 3600 if len(ts) > 1 else 1 def episodes(measurements): """Targets rejected day after day: Target issue (a week or more) and Obstruction (less), then the stuck ones.""" min_instants = MIN_HOURS / cadence(measurements) by_t = {} for s, ms in measurements.items(): for m in ms: by_t.setdefault(m["t"], {})[s] = is_rejected(m) days = {} # sensor -> day -> [rejected, calm, first measurement, last] for t in sorted(by_t): d = by_t[t] for s, r in d.items(): others = [v for k, v in d.items() if k != s] c = days.setdefault(s, {}).setdefault(t.date(), [0, 0, t, t]) c[3] = t if others and sum(others) / len(others) <= CALM: c[0] += r c[1] += 1 out = [] for s, ds in sorted(days.items()): covered = [] # bridged runs of at least WEEK days, at SHARE_WEEK of rejections: a problem target for run in _runs(ds, min_instants, BRIDGE, SHARE_WEEK): share = sum(c[0] for c in run) / sum(c[1] for c in run) if len(run) < WEEK or share < SHARE_WEEK: continue n = sum(c[0] / c[1] >= SHARE_WEEK for c in run) r = _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]]) covered.append((run[0][2], run[-1][3])) out.append({"start": iso(r[0][0]), "end": iso(r[-1][1]), "kind": "Target issue", "label": "Degraded target", "text": f"{s} was rejected in {round(100 * share)} % of the measurements when the other targets were " f"seen, on {n} of {len(run)} days.", "ignore": True, "sensors": [s]}) for run in _runs(ds, min_instants, 0, SHARE): # at times: one episode per run of rejected measurements if len(run) < MIN_DAYS or any(a <= run[0][2] <= b for a, b in covered): continue share = round(100 * sum(c[0] for c in run) / sum(c[1] for c in run)) text = f"{s} was rejected in {share} % of the measurements when the other targets were seen, {len(run)} days in a row." out += [{"start": iso(a), "end": iso(b), "kind": "Obstruction", "label": "Target obscured", "text": text, "ignore": True, "sensors": [s]} for a, b in _rejections([m for m in measurements[s] if run[0][2] <= m["t"] <= run[-1][3]])] return sorted(out + stuck(measurements, min_instants), key=lambda e: e["start"]) def _runs(ds, min_instants, bridge, share): """{day: [rejected, calm, first, last]} -> runs of degraded days; at most `bridge` non-degraded days in a row between two degraded days do not break the run (they belong to it); a day that does not count does not break it.""" runs, cur, gap = [], [], [] for j in sorted(ds): rej, calm = ds[j][:2] if calm < min_instants: continue if rej / calm >= share: cur += gap + [ds[j]] gap = [] elif cur: gap.append(ds[j]) if len(gap) > bridge: runs.append(cur) cur, gap = [], [] return runs + ([cur] if cur else []) def _rejections(ms): """Measurements of a sensor -> [(start, end)] of its runs of rejected measurements.""" out, cur = [], [] for m in ms + [None]: if m and is_rejected(m): cur.append(m["t"]) elif cur: out.append((cur[0], cur[-1])) cur = [] return out def _noise(v): return statistics.median(abs(y - x) for x, y in zip(v, v[1:])) if len(v) > 2 else None def stuck(measurements, min_instants): """Targets whose corr_x and corr_y barely move, day after day, while the others move.""" days = {} # day -> target -> (x noise, y noise, first measurement, last) for s, ms in measurements.items(): if is_reference(ms): continue by_day = {} for m in ms: by_day.setdefault(m["t"].date(), []).append(m) for d, dm in by_day.items(): b = [_noise([m[c] for m in dm if m[c] is not None]) for c in ("corr_x", "corr_y")] if None not in b and len(dm) >= min_instants: days.setdefault(d, {})[s] = (*b, dm[0]["t"], dm[-1]["t"]) runs = {} # target -> [[stuck days in a row]] for d in sorted(days): cs = days[d] for s, (bx, by, a, b) in cs.items(): others = [v for k, (x, y, *_) in cs.items() if k != s for v in (x, y)] if len(others) < 4: continue ref = statistics.median(others) ru = runs.setdefault(s, [[]]) if ref and max(bx, by) <= STUCK * ref: ru[-1].append((a, b)) elif ru[-1]: ru.append([]) return [{"start": iso(k[0][0]), "end": iso(k[-1][1]), "kind": "Target issue", "label": "Target stuck", "text": f"{s} barely moved while the other targets did ({len(k)} day{'s' if len(k) > 1 else ''}): it may be " "set on a reference prism.", "ignore": True, "sensors": [s]} for s, ru in sorted(runs.items()) for k in ru if k] def detect(measurements): return sorted((e for e in episodes(measurements) + displacement.episodes(measurements) if e["kind"] == "Target issue"), key=lambda e: e["start"]) if __name__ == "__main__": for e in detect(load(sys.argv[1])): print(e)