Auto labels reviewed per station and cross-reviewed; all labeling knowledge in labeling/ (D-130)
- Every station's auto labels reviewed by an analyst (/review-auto-labels), then a cross-review per family across all stations (new skill /cross-review-labels), a finishing pass, and descriptions rewritten in words (no numeric values). - The three never-labeled ranges (WILDSTRUBEL 07/09-01/10, NORDEND 16/08-01/10) labeled and reviewed: 991 auto labels on 11 stations, labeling/controle.py at 0 errors. - Labeling knowledge gathered in labeling/: regles-d-etiquetage.md (new), familles.md (settled boundaries), connaissances/ (notes moved from docs/, links updated), the label map rebuilt from reviewed labels, events and cases; events tied to their stations (PS_DES_COUDRES is JUNGFRAU). The labeler and reviewers read only labeling/. - labels-map reads auto labels and events; decision D-130; living plan points 73-75. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8e1cfc52c9
commit
35b5ed4d74
298 files changed
+30413
-960
No files matched your search
@@ -0,0 +1,86 @@
|
||||
"""Trous de mesure de chaque station sur ses lots : tous les instants où aucun capteur n'a de mesure.
|
||||
Par trou : mesure d'avant, d'après, pas manquants, étiquette No data qui le couvre, métriques (BusVoltage, last_sync,
|
||||
DB_not_uploaded) dans le trou et autour, pose de la caméra, images, et les stations qui ont un trou au même moment.
|
||||
Lancé depuis la racine : python3 <ce fichier> > <sortie>/trous.txt"""
|
||||
import csv, glob, json, os, statistics, sys
|
||||
from datetime import datetime, timedelta
|
||||
UTC = lambda s: datetime.fromisoformat(s.replace('Z', '+00:00').replace(' ', 'T')[:16] + '+00:00')
|
||||
LOTS = { # lots donnés à l'etiqueteur
|
||||
'BREITHORN': [('2026-10-01', '2026-11-01')], 'CLARIDEN': [('2026-10-01', '2026-11-01')],
|
||||
'FLORIDA': [('2026-10-01', '2026-11-01')], 'JUNGFRAU': [('2026-09-30', '2026-11-01')],
|
||||
'LENZSPITZE': [('2026-10-01', '2026-11-01')], 'MICHIGAN': [('2026-10-01', '2026-11-01')],
|
||||
'NEBRASKA': [('2026-10-01', '2026-11-01')], 'NIEDERHORN': [('2026-10-01', '2026-11-01')],
|
||||
'NORDEND': [('2026-08-10', '2026-08-16'), ('2026-10-01', '2026-11-01')],
|
||||
'SNCF01': [('2026-10-01', '2026-11-01')],
|
||||
'WILDSTRUBEL': [('2026-09-01', '2026-09-07'), ('2026-10-01', '2026-11-01')],
|
||||
}
|
||||
HORS = {'WILDSTRUBEL': ('2026-09-07', '2026-10-01'), 'NORDEND': ('2026-08-16', '2026-10-01')}
|
||||
def d(s): return datetime.fromisoformat(s + 'T00:00+00:00')
|
||||
def jours(a, b):
|
||||
x = a
|
||||
while x < b: yield x; x += timedelta(days=1)
|
||||
def instants(st, a, b):
|
||||
out = set()
|
||||
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
|
||||
p = f'data/{st}/displacement/{st}_advanced_{j:%Y%m%d}.csv'
|
||||
if os.path.exists(p):
|
||||
for r in csv.DictReader(open(p)):
|
||||
out.add(UTC(r['time_utc']))
|
||||
return sorted(out)
|
||||
def metrique(st, nom, a, b):
|
||||
v = {}
|
||||
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
|
||||
for p in glob.glob(f'data/{st}/metrics/{st}_{nom}_{j:%Y_%m_%d}_*.csv'):
|
||||
for r in csv.DictReader(open(p)):
|
||||
if r['metric'] not in ('', 'nan', '[]'): v[UTC(r['time_utc'])] = r['metric']
|
||||
return v
|
||||
def poses(st, a, b):
|
||||
out = set()
|
||||
for j in jours(a - timedelta(days=1), b + timedelta(days=1)):
|
||||
p = f'data/{st}/camera/{st}_camera_{j:%Y%m%d}.csv'
|
||||
if os.path.exists(p):
|
||||
for r in csv.DictReader(open(p)): out.add(UTC(r['time_utc']))
|
||||
return out
|
||||
def images(st):
|
||||
return {UTC(f[:4] + '-' + f[4:6] + '-' + f[6:8] + 'T' + f[9:11] + ':' + f[11:13] + 'Z')
|
||||
for f in os.listdir(f'data/{st}/images') if f[:8].isdigit()}
|
||||
def etiquettes(st):
|
||||
es = []
|
||||
for n, l in enumerate(open(f'labeling/auto/{st}.jsonl'), 1):
|
||||
if l.strip():
|
||||
e = json.loads(l); es.append((f'{st}#{n}', UTC(e['debut']), UTC(e['fin']), e['categorie']))
|
||||
return es
|
||||
def h(t): return f'{t:%m-%d %H:%M}'
|
||||
tous = {}
|
||||
for st, lots in LOTS.items():
|
||||
ims, es = images(st), etiquettes(st)
|
||||
for a, b in lots:
|
||||
a, b = d(a), d(b)
|
||||
ts = instants(st, a, b)
|
||||
pas = statistics.median([(y - x).total_seconds() for x, y in zip(ts, ts[1:])])
|
||||
bv, ls, nu = (metrique(st, n, a, b) for n in ('BusVoltage', 'last_sync', 'DB_not_uploaded'))
|
||||
ps = poses(st, a, b)
|
||||
print(f'\n=== {st} {a:%m-%d} -> {b:%m-%d} pas {pas/60:.0f} min, {len([t for t in ts if a <= t < b])} instants, '
|
||||
f'BusVoltage {len([t for t in bv if a <= t < b])} h, last_sync {len([t for t in ls if a <= t < b])} h')
|
||||
for x, y in zip(ts, ts[1:]):
|
||||
dt = (y - x).total_seconds()
|
||||
if dt <= 1.5 * pas or y <= a or x >= b: continue
|
||||
manq = round(dt / pas) - 1
|
||||
# heures pleines strictement dans le trou (métriques horaires datées au début de l'heure)
|
||||
hrs = [x.replace(minute=0) + timedelta(hours=k) for k in range(1, int(dt // 3600) + 1)
|
||||
if x.replace(minute=0) + timedelta(hours=k) + timedelta(hours=1) <= y + timedelta(minutes=59)]
|
||||
hrs = [t for t in hrs if x < t + timedelta(minutes=59) and t < y]
|
||||
bvin = sum(t in bv for t in hrs); lsin = sum(t in ls for t in hrs)
|
||||
av = bv.get(x.replace(minute=0), '-'); ap = bv.get(y.replace(minute=0), '-')
|
||||
nuin = [nu[t] for t in hrs if t in nu]
|
||||
imin = sum(x < t < y for t in ims); psin = sum(x < t < y for t in ps)
|
||||
lab = [i for i, ld, lf, c in es if c == 'No data' and ld <= x + timedelta(minutes=pas / 120) and lf >= y - timedelta(minutes=pas / 120)]
|
||||
lab = lab or [f'~{i}({c})' for i, ld, lf, c in es if ld < y and lf > x and c == 'No data'] or ['-']
|
||||
tous.setdefault(st, []).append((x, y))
|
||||
print(f'{h(x)} -> {h(y)} manq {manq:4d} heures {len(hrs):3d} BV {bvin}/{len(hrs)} ({str(av)[:5]} | {str(ap)[:5]}) '
|
||||
f'sync {lsin}/{len(hrs)} img {imin} pose {psin} notupl {",".join(str(v)[:5] for v in nuin[:4]) or "-"} {",".join(lab)}')
|
||||
print('\n=== trous simultanés (chevauchement)')
|
||||
for st, l in tous.items():
|
||||
for x, y in l:
|
||||
o = sorted(s for s, m in tous.items() if s != st and any(u < y and v > x for u, v in m))
|
||||
print(st, h(x), h(y), ','.join(o) or 'aucune')
|
||||
Reference in new issue
Block a user