"""Trous de mesure de chaque station sur ses lots : tous les instants où aucun capteur n'a de mesure. Par trou : mesure d'avant, d'après, pas manquants, étiquette No data qui le couvre, métriques (BusVoltage, last_sync, DB_not_uploaded) dans le trou et autour, pose de la caméra, images, et les stations qui ont un trou au même moment. Lancé depuis la racine : python3 > /trous.txt""" import csv, glob, json, os, statistics, sys from datetime import datetime, timedelta UTC = lambda s: datetime.fromisoformat(s.replace('Z', '+00:00').replace(' ', 'T')[:16] + '+00:00') LOTS = { # lots donnés à l'etiqueteur 'BREITHORN': [('2026-10-01', '2026-11-01')], 'CLARIDEN': [('2026-10-01', '2026-11-01')], 'FLORIDA': [('2026-10-01', '2026-11-01')], 'JUNGFRAU': [('2026-09-30', '2026-11-01')], 'LENZSPITZE': [('2026-10-01', '2026-11-01')], 'MICHIGAN': [('2026-10-01', '2026-11-01')], 'NEBRASKA': [('2026-10-01', '2026-11-01')], 'NIEDERHORN': [('2026-10-01', '2026-11-01')], 'NORDEND': [('2026-08-10', '2026-08-16'), ('2026-10-01', '2026-11-01')], 'SNCF01': [('2026-10-01', '2026-11-01')], 'WILDSTRUBEL': [('2026-09-01', '2026-09-07'), ('2026-10-01', '2026-11-01')], } HORS = {'WILDSTRUBEL': ('2026-09-07', '2026-10-01'), 'NORDEND': ('2026-08-16', '2026-10-01')} def d(s): return datetime.fromisoformat(s + 'T00:00+00:00') def jours(a, b): x = a while x < b: yield x; x += timedelta(days=1) def instants(st, a, b): out = set() for j in jours(a - timedelta(days=1), b + timedelta(days=1)): p = f'data/{st}/displacement/{st}_advanced_{j:%Y%m%d}.csv' if os.path.exists(p): for r in csv.DictReader(open(p)): out.add(UTC(r['time_utc'])) return sorted(out) def metrique(st, nom, a, b): v = {} for j in jours(a - timedelta(days=1), b + timedelta(days=1)): for p in glob.glob(f'data/{st}/metrics/{st}_{nom}_{j:%Y_%m_%d}_*.csv'): for r in csv.DictReader(open(p)): if r['metric'] not in ('', 'nan', '[]'): v[UTC(r['time_utc'])] = r['metric'] return v def poses(st, a, b): out = set() for j in jours(a - timedelta(days=1), b + timedelta(days=1)): p = f'data/{st}/camera/{st}_camera_{j:%Y%m%d}.csv' if os.path.exists(p): for r in csv.DictReader(open(p)): out.add(UTC(r['time_utc'])) return out def images(st): return {UTC(f[:4] + '-' + f[4:6] + '-' + f[6:8] + 'T' + f[9:11] + ':' + f[11:13] + 'Z') for f in os.listdir(f'data/{st}/images') if f[:8].isdigit()} def etiquettes(st): es = [] for n, l in enumerate(open(f'labeling/auto/{st}.jsonl'), 1): if l.strip(): e = json.loads(l); es.append((f'{st}#{n}', UTC(e['debut']), UTC(e['fin']), e['categorie'])) return es def h(t): return f'{t:%m-%d %H:%M}' tous = {} for st, lots in LOTS.items(): ims, es = images(st), etiquettes(st) for a, b in lots: a, b = d(a), d(b) ts = instants(st, a, b) pas = statistics.median([(y - x).total_seconds() for x, y in zip(ts, ts[1:])]) bv, ls, nu = (metrique(st, n, a, b) for n in ('BusVoltage', 'last_sync', 'DB_not_uploaded')) ps = poses(st, a, b) print(f'\n=== {st} {a:%m-%d} -> {b:%m-%d} pas {pas/60:.0f} min, {len([t for t in ts if a <= t < b])} instants, ' f'BusVoltage {len([t for t in bv if a <= t < b])} h, last_sync {len([t for t in ls if a <= t < b])} h') for x, y in zip(ts, ts[1:]): dt = (y - x).total_seconds() if dt <= 1.5 * pas or y <= a or x >= b: continue manq = round(dt / pas) - 1 # heures pleines strictement dans le trou (métriques horaires datées au début de l'heure) hrs = [x.replace(minute=0) + timedelta(hours=k) for k in range(1, int(dt // 3600) + 1) if x.replace(minute=0) + timedelta(hours=k) + timedelta(hours=1) <= y + timedelta(minutes=59)] hrs = [t for t in hrs if x < t + timedelta(minutes=59) and t < y] bvin = sum(t in bv for t in hrs); lsin = sum(t in ls for t in hrs) av = bv.get(x.replace(minute=0), '-'); ap = bv.get(y.replace(minute=0), '-') nuin = [nu[t] for t in hrs if t in nu] imin = sum(x < t < y for t in ims); psin = sum(x < t < y for t in ps) lab = [i for i, ld, lf, c in es if c == 'No data' and ld <= x + timedelta(minutes=pas / 120) and lf >= y - timedelta(minutes=pas / 120)] lab = lab or [f'~{i}({c})' for i, ld, lf, c in es if ld < y and lf > x and c == 'No data'] or ['-'] tous.setdefault(st, []).append((x, y)) print(f'{h(x)} -> {h(y)} manq {manq:4d} heures {len(hrs):3d} BV {bvin}/{len(hrs)} ({str(av)[:5]} | {str(ap)[:5]}) ' f'sync {lsin}/{len(hrs)} img {imin} pose {psin} notupl {",".join(str(v)[:5] for v in nuin[:4]) or "-"} {",".join(lab)}') print('\n=== trous simultanés (chevauchement)') for st, l in tous.items(): for x, y in l: o = sorted(s for s, m in tous.items() if s != st and any(u < y and v > x for u, v in m)) print(st, h(x), h(y), ','.join(o) or 'aucune')