- Every station's auto labels reviewed by an analyst (/review-auto-labels), then a cross-review per family across all stations (new skill /cross-review-labels), a finishing pass, and descriptions rewritten in words (no numeric values). - The three never-labeled ranges (WILDSTRUBEL 07/09-01/10, NORDEND 16/08-01/10) labeled and reviewed: 991 auto labels on 11 stations, labeling/controle.py at 0 errors. - Labeling knowledge gathered in labeling/: regles-d-etiquetage.md (new), familles.md (settled boundaries), connaissances/ (notes moved from docs/, links updated), the label map rebuilt from reviewed labels, events and cases; events tied to their stations (PS_DES_COUDRES is JUNGFRAU). The labeler and reviewers read only labeling/. - labels-map reads auto labels and events; decision D-130; living plan points 73-75. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
4.6 KiB
Python
79 lines
4.6 KiB
Python
"""Bornes des étiquettes Obstruction face aux instants de mesure de leur station.
|
|
Pour chaque étiquette : debut et fin sont-ils des instants de mesure, combien d'instants entre les deux,
|
|
et, aux instants prev(debut), debut, fin, next(fin) : les capteurs rejetés et ceux qui s'écartent
|
|
(|corr - médiane du jour| > 6 MAD), pour voir si une borne est elle-même touchée.
|
|
Usage (racine du dépôt) : python3 .../outils/bornes.py [STATION ...] > .../obstruction/bornes.txt"""
|
|
import csv, glob, json, sys, statistics as S
|
|
from datetime import datetime, timedelta
|
|
from collections import defaultdict
|
|
|
|
def t(s): return datetime.fromisoformat(s.replace('Z', '+00:00'))
|
|
|
|
def charge(st, d0, d1):
|
|
rows = defaultdict(dict) # instant -> capteur -> (ev, thr, cx, cy)
|
|
d = d0.date() - timedelta(days=1)
|
|
while d <= d1.date() + timedelta(days=1):
|
|
for p in glob.glob(f'data/{st}/displacement/{st}_advanced_{d:%Y%m%d}*.csv'):
|
|
for r in csv.DictReader(open(p)):
|
|
i = datetime.fromisoformat(r['time_utc']).replace(second=0, microsecond=0)
|
|
f = lambda k: float(r[k]) if r[k] not in ('', None) else None
|
|
rows[i][r['sensor_name'].strip()] = (f('error_value'), f('error_threshold'), f('corr_x'), f('corr_y'))
|
|
d += timedelta(days=1)
|
|
return rows
|
|
|
|
def ecarts(rows, inst):
|
|
"""capteurs dont corr s'écarte de plus de 6 MAD de sa médiane sur la fenêtre chargée"""
|
|
par = defaultdict(lambda: ([], []))
|
|
for i in rows:
|
|
for c, (ev, th, cx, cy) in rows[i].items():
|
|
if cx is not None: par[c][0].append(cx)
|
|
if cy is not None: par[c][1].append(cy)
|
|
ref = {}
|
|
for c, (xs, ys) in par.items():
|
|
ref[c] = [(S.median(v), S.median([abs(a - S.median(v)) for a in v]) or 1e-9) if len(v) > 5 else None for v in (xs, ys)]
|
|
out = {}
|
|
for i in inst:
|
|
s = set()
|
|
for c, (ev, th, cx, cy) in rows.get(i, {}).items():
|
|
for k, v in enumerate((cx, cy)):
|
|
if v is not None and ref.get(c) and ref[c][k] and abs(v - ref[c][k][0]) > 6 * ref[c][k][1]:
|
|
s.add(c)
|
|
out[i] = s
|
|
return out
|
|
|
|
def rej(rows, i):
|
|
return {c for c, (ev, th, cx, cy) in rows.get(i, {}).items() if ev is not None and th is not None and ev > th}
|
|
|
|
stations = sys.argv[1:] or ['BREITHORN', 'CLARIDEN', 'FLORIDA', 'JUNGFRAU', 'LENZSPITZE', 'NIEDERHORN', 'NORDEND', 'SNCF01', 'WILDSTRUBEL']
|
|
for st in stations:
|
|
for n, l in enumerate(open(f'labeling/auto/{st}.jsonl'), 1):
|
|
if not l.strip(): continue
|
|
e = json.loads(l)
|
|
if e['categorie'] != 'Obstruction': continue
|
|
d0, d1 = t(e['debut']), t(e['fin'])
|
|
if (d1 - d0) > timedelta(hours=30):
|
|
print(f'{st}#{n} {e["debut"]} -> {e["fin"]} : longue, non vérifiée'); continue
|
|
rows = charge(st, d0, d1)
|
|
inst = sorted(rows)
|
|
dedans = [i for i in inst if d0 < i < d1]
|
|
avant = [i for i in inst if i < d0][-1:] ; apres = [i for i in inst if i > d1][:1]
|
|
pas = S.median([(b - a).total_seconds() / 60 for a, b in zip(inst, inst[1:])]) if len(inst) > 2 else None
|
|
cle = [*avant, *(i for i in inst if i == d0), *dedans, *(i for i in inst if i == d1), *apres]
|
|
ec = ecarts(rows, cle)
|
|
flag = []
|
|
if d0 not in rows: flag.append('debut hors instant')
|
|
if d1 not in rows: flag.append('fin hors instant')
|
|
if d0 >= d1: flag.append('debut>=fin')
|
|
if not dedans: flag.append('aucun instant dedans')
|
|
R = lambda i: rej(rows, i)
|
|
base = R(avant[0]) if avant else set()
|
|
if d0 in rows and R(d0) - base: flag.append('DEBUT TOUCHE (rejets) ' + ','.join(sorted(R(d0) - base))[:60])
|
|
if d1 in rows and apres and R(d1) - R(apres[0]) - base: flag.append('FIN TOUCHEE (rejets) ' + ','.join(sorted(R(d1) - R(apres[0]) - base))[:60])
|
|
if d1 in rows and len(ec[d1] - R(d1) - (ec[apres[0]] if apres else set())) >= 2: flag.append('fin: ecarts ' + ','.join(sorted(ec[d1] - R(d1) - (ec[apres[0]] if apres else set())))[:60])
|
|
if d0 in rows and avant and len(ec[d0] - R(d0) - ec[avant[0]]) >= 2: flag.append('debut: ecarts ' + ','.join(sorted(ec[d0] - R(d0) - ec[avant[0]]))[:60])
|
|
print(f'{st}#{n} {e["debut"]} -> {e["fin"]} pas~{pas}min, {len(dedans)} instants dedans {" ; ".join(flag)}')
|
|
for i in cle:
|
|
tag = 'avant' if i in avant else 'DEBUT' if i == d0 else 'FIN' if i == d1 else 'apres' if i in apres else 'dedans'
|
|
r = rej(rows, i); x = ec[i] - r
|
|
print(f' {i:%m-%d %H:%M} {tag:6} rejetes {len(r):2} ecarts {len(x):2} {",".join(sorted(r))[:110]} | {",".join(sorted(x))[:80]}')
|