- Every station's auto labels reviewed by an analyst (/review-auto-labels), then a cross-review per family across all stations (new skill /cross-review-labels), a finishing pass, and descriptions rewritten in words (no numeric values). - The three never-labeled ranges (WILDSTRUBEL 07/09-01/10, NORDEND 16/08-01/10) labeled and reviewed: 991 auto labels on 11 stations, labeling/controle.py at 0 errors. - Labeling knowledge gathered in labeling/: regles-d-etiquetage.md (new), familles.md (settled boundaries), connaissances/ (notes moved from docs/, links updated), the label map rebuilt from reviewed labels, events and cases; events tied to their stations (PS_DES_COUDRES is JUNGFRAU). The labeler and reviewers read only labeling/. - labels-map reads auto labels and events; decision D-130; living plan points 73-75. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
5.0 KiB
Python
87 lines
5.0 KiB
Python
"""Trous de mesure de chaque station sur ses lots : tous les instants où aucun capteur n'a de mesure.
|
|
Par trou : mesure d'avant, d'après, pas manquants, étiquette No data qui le couvre, métriques (BusVoltage, last_sync,
|
|
DB_not_uploaded) dans le trou et autour, pose de la caméra, images, et les stations qui ont un trou au même moment.
|
|
Lancé depuis la racine : python3 <ce fichier> > <sortie>/trous.txt"""
|
|
import csv, glob, json, os, statistics, sys
|
|
from datetime import datetime, timedelta
|
|
UTC = lambda s: datetime.fromisoformat(s.replace('Z', '+00:00').replace(' ', 'T')[:16] + '+00:00')
|
|
LOTS = { # lots donnés à l'etiqueteur
|
|
'BREITHORN': [('2026-10-01', '2026-11-01')], 'CLARIDEN': [('2026-10-01', '2026-11-01')],
|
|
'FLORIDA': [('2026-10-01', '2026-11-01')], 'JUNGFRAU': [('2026-09-30', '2026-11-01')],
|
|
'LENZSPITZE': [('2026-10-01', '2026-11-01')], 'MICHIGAN': [('2026-10-01', '2026-11-01')],
|
|
'NEBRASKA': [('2026-10-01', '2026-11-01')], 'NIEDERHORN': [('2026-10-01', '2026-11-01')],
|
|
'NORDEND': [('2026-08-10', '2026-08-16'), ('2026-10-01', '2026-11-01')],
|
|
'SNCF01': [('2026-10-01', '2026-11-01')],
|
|
'WILDSTRUBEL': [('2026-09-01', '2026-09-07'), ('2026-10-01', '2026-11-01')],
|
|
}
|
|
HORS = {'WILDSTRUBEL': ('2026-09-07', '2026-10-01'), 'NORDEND': ('2026-08-16', '2026-10-01')}
|
|
def d(s): return datetime.fromisoformat(s + 'T00:00+00:00')
|
|
def jours(a, b):
|
|
x = a
|
|
while x < b: yield x; x += timedelta(days=1)
|
|
def instants(st, a, b):
|
|
out = set()
|
|
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
|
|
p = f'data/{st}/displacement/{st}_advanced_{j:%Y%m%d}.csv'
|
|
if os.path.exists(p):
|
|
for r in csv.DictReader(open(p)):
|
|
out.add(UTC(r['time_utc']))
|
|
return sorted(out)
|
|
def metrique(st, nom, a, b):
|
|
v = {}
|
|
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
|
|
for p in glob.glob(f'data/{st}/metrics/{st}_{nom}_{j:%Y_%m_%d}_*.csv'):
|
|
for r in csv.DictReader(open(p)):
|
|
if r['metric'] not in ('', 'nan', '[]'): v[UTC(r['time_utc'])] = r['metric']
|
|
return v
|
|
def poses(st, a, b):
|
|
out = set()
|
|
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
|
|
p = f'data/{st}/camera/{st}_camera_{j:%Y%m%d}.csv'
|
|
if os.path.exists(p):
|
|
for r in csv.DictReader(open(p)): out.add(UTC(r['time_utc']))
|
|
return out
|
|
def images(st):
|
|
return {UTC(f[:4] + '-' + f[4:6] + '-' + f[6:8] + 'T' + f[9:11] + ':' + f[11:13] + 'Z')
|
|
for f in os.listdir(f'data/{st}/images') if f[:8].isdigit()}
|
|
def etiquettes(st):
|
|
es = []
|
|
for n, l in enumerate(open(f'labeling/auto/{st}.jsonl'), 1):
|
|
if l.strip():
|
|
e = json.loads(l); es.append((f'{st}#{n}', UTC(e['debut']), UTC(e['fin']), e['categorie']))
|
|
return es
|
|
def h(t): return f'{t:%m-%d %H:%M}'
|
|
tous = {}
|
|
for st, lots in LOTS.items():
|
|
ims, es = images(st), etiquettes(st)
|
|
for a, b in lots:
|
|
a, b = d(a), d(b)
|
|
ts = instants(st, a, b)
|
|
pas = statistics.median([(y - x).total_seconds() for x, y in zip(ts, ts[1:])])
|
|
bv, ls, nu = (metrique(st, n, a, b) for n in ('BusVoltage', 'last_sync', 'DB_not_uploaded'))
|
|
ps = poses(st, a, b)
|
|
print(f'\n=== {st} {a:%m-%d} -> {b:%m-%d} pas {pas/60:.0f} min, {len([t for t in ts if a <= t < b])} instants, '
|
|
f'BusVoltage {len([t for t in bv if a <= t < b])} h, last_sync {len([t for t in ls if a <= t < b])} h')
|
|
for x, y in zip(ts, ts[1:]):
|
|
dt = (y - x).total_seconds()
|
|
if dt <= 1.5 * pas or y <= a or x >= b: continue
|
|
manq = round(dt / pas) - 1
|
|
# heures pleines strictement dans le trou (métriques horaires datées au début de l'heure)
|
|
hrs = [x.replace(minute=0) + timedelta(hours=k) for k in range(1, int(dt // 3600) + 1)
|
|
if x.replace(minute=0) + timedelta(hours=k) + timedelta(hours=1) <= y + timedelta(minutes=59)]
|
|
hrs = [t for t in hrs if x < t + timedelta(minutes=59) and t < y]
|
|
bvin = sum(t in bv for t in hrs); lsin = sum(t in ls for t in hrs)
|
|
av = bv.get(x.replace(minute=0), '-'); ap = bv.get(y.replace(minute=0), '-')
|
|
nuin = [nu[t] for t in hrs if t in nu]
|
|
imin = sum(x < t < y for t in ims); psin = sum(x < t < y for t in ps)
|
|
lab = [i for i, ld, lf, c in es if c == 'No data' and ld <= x + timedelta(minutes=pas / 120) and lf >= y - timedelta(minutes=pas / 120)]
|
|
lab = lab or [f'~{i}({c})' for i, ld, lf, c in es if ld < y and lf > x and c == 'No data'] or ['-']
|
|
tous.setdefault(st, []).append((x, y))
|
|
print(f'{h(x)} -> {h(y)} manq {manq:4d} heures {len(hrs):3d} BV {bvin}/{len(hrs)} ({str(av)[:5]} | {str(ap)[:5]}) '
|
|
f'sync {lsin}/{len(hrs)} img {imin} pose {psin} notupl {",".join(str(v)[:5] for v in nuin[:4]) or "-"} {",".join(lab)}')
|
|
print('\n=== trous simultanés (chevauchement)')
|
|
for st, l in tous.items():
|
|
for x, y in l:
|
|
o = sorted(s for s, m in tous.items() if s != st and any(u < y and v > x for u, v in m))
|
|
print(st, h(x), h(y), ','.join(o) or 'aucune')
|