"""Bornes des étiquettes Obstruction face aux instants de mesure de leur station. Pour chaque étiquette : debut et fin sont-ils des instants de mesure, combien d'instants entre les deux, et, aux instants prev(debut), debut, fin, next(fin) : les capteurs rejetés et ceux qui s'écartent (|corr - médiane du jour| > 6 MAD), pour voir si une borne est elle-même touchée. Usage (racine du dépôt) : python3 .../outils/bornes.py [STATION ...] > .../obstruction/bornes.txt""" import csv, glob, json, sys, statistics as S from datetime import datetime, timedelta from collections import defaultdict def t(s): return datetime.fromisoformat(s.replace('Z', '+00:00')) def charge(st, d0, d1): rows = defaultdict(dict) # instant -> capteur -> (ev, thr, cx, cy) d = d0.date() - timedelta(days=1) while d <= d1.date() + timedelta(days=1): for p in glob.glob(f'data/{st}/displacement/{st}_advanced_{d:%Y%m%d}*.csv'): for r in csv.DictReader(open(p)): i = datetime.fromisoformat(r['time_utc']).replace(second=0, microsecond=0) f = lambda k: float(r[k]) if r[k] not in ('', None) else None rows[i][r['sensor_name'].strip()] = (f('error_value'), f('error_threshold'), f('corr_x'), f('corr_y')) d += timedelta(days=1) return rows def ecarts(rows, inst): """capteurs dont corr s'écarte de plus de 6 MAD de sa médiane sur la fenêtre chargée""" par = defaultdict(lambda: ([], [])) for i in rows: for c, (ev, th, cx, cy) in rows[i].items(): if cx is not None: par[c][0].append(cx) if cy is not None: par[c][1].append(cy) ref = {} for c, (xs, ys) in par.items(): ref[c] = [(S.median(v), S.median([abs(a - S.median(v)) for a in v]) or 1e-9) if len(v) > 5 else None for v in (xs, ys)] out = {} for i in inst: s = set() for c, (ev, th, cx, cy) in rows.get(i, {}).items(): for k, v in enumerate((cx, cy)): if v is not None and ref.get(c) and ref[c][k] and abs(v - ref[c][k][0]) > 6 * ref[c][k][1]: s.add(c) out[i] = s return out def rej(rows, i): return {c for c, (ev, th, cx, cy) in rows.get(i, {}).items() if ev is not None and th is not None and ev > th} stations = sys.argv[1:] or ['BREITHORN', 'CLARIDEN', 'FLORIDA', 'JUNGFRAU', 'LENZSPITZE', 'NIEDERHORN', 'NORDEND', 'SNCF01', 'WILDSTRUBEL'] for st in stations: for n, l in enumerate(open(f'labeling/auto/{st}.jsonl'), 1): if not l.strip(): continue e = json.loads(l) if e['categorie'] != 'Obstruction': continue d0, d1 = t(e['debut']), t(e['fin']) if (d1 - d0) > timedelta(hours=30): print(f'{st}#{n} {e["debut"]} -> {e["fin"]} : longue, non vérifiée'); continue rows = charge(st, d0, d1) inst = sorted(rows) dedans = [i for i in inst if d0 < i < d1] avant = [i for i in inst if i < d0][-1:] ; apres = [i for i in inst if i > d1][:1] pas = S.median([(b - a).total_seconds() / 60 for a, b in zip(inst, inst[1:])]) if len(inst) > 2 else None cle = [*avant, *(i for i in inst if i == d0), *dedans, *(i for i in inst if i == d1), *apres] ec = ecarts(rows, cle) flag = [] if d0 not in rows: flag.append('debut hors instant') if d1 not in rows: flag.append('fin hors instant') if d0 >= d1: flag.append('debut>=fin') if not dedans: flag.append('aucun instant dedans') R = lambda i: rej(rows, i) base = R(avant[0]) if avant else set() if d0 in rows and R(d0) - base: flag.append('DEBUT TOUCHE (rejets) ' + ','.join(sorted(R(d0) - base))[:60]) if d1 in rows and apres and R(d1) - R(apres[0]) - base: flag.append('FIN TOUCHEE (rejets) ' + ','.join(sorted(R(d1) - R(apres[0]) - base))[:60]) if d1 in rows and len(ec[d1] - R(d1) - (ec[apres[0]] if apres else set())) >= 2: flag.append('fin: ecarts ' + ','.join(sorted(ec[d1] - R(d1) - (ec[apres[0]] if apres else set())))[:60]) if d0 in rows and avant and len(ec[d0] - R(d0) - ec[avant[0]]) >= 2: flag.append('debut: ecarts ' + ','.join(sorted(ec[d0] - R(d0) - ec[avant[0]]))[:60]) print(f'{st}#{n} {e["debut"]} -> {e["fin"]} pas~{pas}min, {len(dedans)} instants dedans {" ; ".join(flag)}') for i in cle: tag = 'avant' if i in avant else 'DEBUT' if i == d0 else 'FIN' if i == d1 else 'apres' if i in apres else 'dedans' r = rej(rows, i); x = ec[i] - r print(f' {i:%m-%d %H:%M} {tag:6} rejetes {len(r):2} ecarts {len(x):2} {",".join(sorted(r))[:110]} | {",".join(sorted(x))[:80]}')