Files
Support-Agent/labeling/auto/revue-croisee/2026-11-04/no-data/outils/trous.py
T
maximeandClaude Opus 5.5 35b5ed4d74 Auto labels reviewed per station and cross-reviewed; all labeling knowledge in labeling/ (D-130)
- Every station's auto labels reviewed by an analyst (/review-auto-labels), then a
  cross-review per family across all stations (new skill /cross-review-labels),
  a finishing pass, and descriptions rewritten in words (no numeric values).
- The three never-labeled ranges (WILDSTRUBEL 07/09-01/10, NORDEND 16/08-01/10)
  labeled and reviewed: 991 auto labels on 11 stations, labeling/controle.py at 0 errors.
- Labeling knowledge gathered in labeling/: regles-d-etiquetage.md (new), familles.md
  (settled boundaries), connaissances/ (notes moved from docs/, links updated),
  the label map rebuilt from reviewed labels, events and cases; events tied to their
  stations (PS_DES_COUDRES is JUNGFRAU). The labeler and reviewers read only labeling/.
- labels-map reads auto labels and events; decision D-130; living plan points 73-75.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 08:21:57 +01:00

87 lines
5.0 KiB
Python

"""Trous de mesure de chaque station sur ses lots : tous les instants où aucun capteur n'a de mesure.
Par trou : mesure d'avant, d'après, pas manquants, étiquette No data qui le couvre, métriques (BusVoltage, last_sync,
DB_not_uploaded) dans le trou et autour, pose de la caméra, images, et les stations qui ont un trou au même moment.
Lancé depuis la racine : python3 <ce fichier> > <sortie>/trous.txt"""
import csv, glob, json, os, statistics, sys
from datetime import datetime, timedelta
UTC = lambda s: datetime.fromisoformat(s.replace('Z', '+00:00').replace(' ', 'T')[:16] + '+00:00')
LOTS = { # lots donnés à l'etiqueteur
'BREITHORN': [('2026-10-01', '2026-11-01')], 'CLARIDEN': [('2026-10-01', '2026-11-01')],
'FLORIDA': [('2026-10-01', '2026-11-01')], 'JUNGFRAU': [('2026-09-30', '2026-11-01')],
'LENZSPITZE': [('2026-10-01', '2026-11-01')], 'MICHIGAN': [('2026-10-01', '2026-11-01')],
'NEBRASKA': [('2026-10-01', '2026-11-01')], 'NIEDERHORN': [('2026-10-01', '2026-11-01')],
'NORDEND': [('2026-08-10', '2026-08-16'), ('2026-10-01', '2026-11-01')],
'SNCF01': [('2026-10-01', '2026-11-01')],
'WILDSTRUBEL': [('2026-09-01', '2026-09-07'), ('2026-10-01', '2026-11-01')],
}
HORS = {'WILDSTRUBEL': ('2026-09-07', '2026-10-01'), 'NORDEND': ('2026-08-16', '2026-10-01')}
def d(s): return datetime.fromisoformat(s + 'T00:00+00:00')
def jours(a, b):
x = a
while x < b: yield x; x += timedelta(days=1)
def instants(st, a, b):
out = set()
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
p = f'data/{st}/displacement/{st}_advanced_{j:%Y%m%d}.csv'
if os.path.exists(p):
for r in csv.DictReader(open(p)):
out.add(UTC(r['time_utc']))
return sorted(out)
def metrique(st, nom, a, b):
v = {}
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
for p in glob.glob(f'data/{st}/metrics/{st}_{nom}_{j:%Y_%m_%d}_*.csv'):
for r in csv.DictReader(open(p)):
if r['metric'] not in ('', 'nan', '[]'): v[UTC(r['time_utc'])] = r['metric']
return v
def poses(st, a, b):
out = set()
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
p = f'data/{st}/camera/{st}_camera_{j:%Y%m%d}.csv'
if os.path.exists(p):
for r in csv.DictReader(open(p)): out.add(UTC(r['time_utc']))
return out
def images(st):
return {UTC(f[:4] + '-' + f[4:6] + '-' + f[6:8] + 'T' + f[9:11] + ':' + f[11:13] + 'Z')
for f in os.listdir(f'data/{st}/images') if f[:8].isdigit()}
def etiquettes(st):
es = []
for n, l in enumerate(open(f'labeling/auto/{st}.jsonl'), 1):
if l.strip():
e = json.loads(l); es.append((f'{st}#{n}', UTC(e['debut']), UTC(e['fin']), e['categorie']))
return es
def h(t): return f'{t:%m-%d %H:%M}'
tous = {}
for st, lots in LOTS.items():
ims, es = images(st), etiquettes(st)
for a, b in lots:
a, b = d(a), d(b)
ts = instants(st, a, b)
pas = statistics.median([(y - x).total_seconds() for x, y in zip(ts, ts[1:])])
bv, ls, nu = (metrique(st, n, a, b) for n in ('BusVoltage', 'last_sync', 'DB_not_uploaded'))
ps = poses(st, a, b)
print(f'\n=== {st} {a:%m-%d} -> {b:%m-%d} pas {pas/60:.0f} min, {len([t for t in ts if a <= t < b])} instants, '
f'BusVoltage {len([t for t in bv if a <= t < b])} h, last_sync {len([t for t in ls if a <= t < b])} h')
for x, y in zip(ts, ts[1:]):
dt = (y - x).total_seconds()
if dt <= 1.5 * pas or y <= a or x >= b: continue
manq = round(dt / pas) - 1
# heures pleines strictement dans le trou (métriques horaires datées au début de l'heure)
hrs = [x.replace(minute=0) + timedelta(hours=k) for k in range(1, int(dt // 3600) + 1)
if x.replace(minute=0) + timedelta(hours=k) + timedelta(hours=1) <= y + timedelta(minutes=59)]
hrs = [t for t in hrs if x < t + timedelta(minutes=59) and t < y]
bvin = sum(t in bv for t in hrs); lsin = sum(t in ls for t in hrs)
av = bv.get(x.replace(minute=0), '-'); ap = bv.get(y.replace(minute=0), '-')
nuin = [nu[t] for t in hrs if t in nu]
imin = sum(x < t < y for t in ims); psin = sum(x < t < y for t in ps)
lab = [i for i, ld, lf, c in es if c == 'No data' and ld <= x + timedelta(minutes=pas / 120) and lf >= y - timedelta(minutes=pas / 120)]
lab = lab or [f'~{i}({c})' for i, ld, lf, c in es if ld < y and lf > x and c == 'No data'] or ['-']
tous.setdefault(st, []).append((x, y))
print(f'{h(x)} -> {h(y)} manq {manq:4d} heures {len(hrs):3d} BV {bvin}/{len(hrs)} ({str(av)[:5]} | {str(ap)[:5]}) '
f'sync {lsin}/{len(hrs)} img {imin} pose {psin} notupl {",".join(str(v)[:5] for v in nuin[:4]) or "-"} {",".join(lab)}')
print('\n=== trous simultanés (chevauchement)')
for st, l in tous.items():
for x, y in l:
o = sorted(s for s, m in tous.items() if s != st and any(u < y and v > x for u, v in m))
print(st, h(x), h(y), ','.join(o) or 'aucune')