auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 15:13:44 +01:00
1 parent df979301b5
commit 2a5d87659b
279 files changed
+10758 -710

No files matched your search

@@ -0,0 +1,20 @@
"""Per instant, median over the targets of (corr - level of a calm window) / noise, from the summary table."""
import sys, statistics as st
f, a, b = sys.argv[1], *sys.argv[2].split('/')
rows, hdr = [], None
for line in open(f):
p = line.split()
if p and p[0] == 'start': hdr = p[1:]; continue
if hdr and p and p[0].startswith('2026-'): rows.append((p[0], p[1:]))
def val(s):
try: return float(s.rstrip('*o'))
except ValueError: return None
tg = [i for i, s in enumerate(hdr) if not s.startswith('REF4') and s not in ('REFG43N', 'REFPOU22')]
lev, noi = {}, {}
for i in tg:
s = [val(v[i]) for t, v in rows]
noi[i] = st.median(abs(y - x) for x, y in zip(s, s[1:]))
lev[i] = st.median(val(v[i]) for t, v in rows if a <= t[:16] + 'Z' <= b)
for t, v in rows:
d = [(val(v[i]) - lev[i]) / noi[i] for i in tg]
print(t[:16], f"median {st.median(d):+5.1f} n>+2: {sum(x > 2 for x in d):2d} n<-2: {sum(x < -2 for x in d):2d}")
@@ -0,0 +1,18 @@
"""Per sensor, deviation (in own noise) from the median of a calm window, at chosen instants; from a summary table."""
import sys, statistics as st
f, (a, b), inst = sys.argv[1], sys.argv[2].split('/'), sys.argv[3].split(',')
rows, hdr = [], None
for line in open(f):
p = line.split()
if p and p[0] == 'start': hdr = p[1:]; continue
if hdr and p and p[0].startswith('2026-'): rows.append((p[0][:16], p[1:]))
def val(s):
try: return float(s.rstrip('*o'))
except ValueError: return None
print('sensor ' + ' '.join(f"{t[5:]:>12s}" for t in inst))
for i, s in enumerate(hdr):
ser = [val(v[i]) for t, v in rows if val(v[i]) is not None]
noise = st.median(abs(y - x) for x, y in zip(ser, ser[1:])) or 1e-9
lev = st.median(val(v[i]) for t, v in rows if a <= t <= b and val(v[i]) is not None)
d = dict((t, val(v[i])) for t, v in rows)
print(f"{s:11s}" + ' '.join(f"{((d[t] - lev) / noise) if d.get(t) is not None else float('nan'):+12.1f}" for t in inst))
@@ -0,0 +1,25 @@
"""Pose and temperature per instant, with the deviation of each axis from the median of the previous 12:00-20:00 measurements, in noises."""
import sys, statistics as st
rows = []
for line in open(sys.argv[1]):
p = line.rstrip('\n').split('\t')
if p[0].startswith('2026-'): rows.append(p)
# columns: time temp A0 CPU GPU PLL thermal BusVoltage tgt_corr_x(8) tgt_corr_y(9) pan tilt rot scale
cols = {'temp': 1, 'cx': 8, 'cy': 9, 'pan': 10, 'tilt': 11, 'rot': 12, 'scale': 13}
def f(s):
try: return float(s)
except ValueError: return None
lim = sys.argv[2] # noise computed before this instant
noise = {}
for k, i in cols.items():
s = [f(r[i]) for r in rows if r[0] < lim]
noise[k] = st.median(abs(b - a) for a, b in zip(s, s[1:]))
print('noise', {k: f"{v:.3g}" for k, v in noise.items()})
for j, r in enumerate(rows):
prev = [q for q in rows[max(0, j-6):j] if q[0][11:13] in ('12', '16', '20')]
if not prev: continue
out = []
for k, i in cols.items():
ref = st.median(f(q[i]) for q in prev)
out.append(f"{k} {(f(r[i]) - ref) / noise[k]:+5.1f}")
print(r[0][:16], r[1], ' '.join(out))
@@ -0,0 +1,14 @@
"""Gap between REF42 and REFG43N raw positions per instant (from the advanced exports), and the common corr of the targets."""
import csv, glob, statistics as st
rows = {}
for f in sorted(glob.glob('data/MICHIGAN/displacement/MICHIGAN_advanced_202610*.csv')):
for r in csv.DictReader(open(f)):
t = r['time_utc'][:16]
if '2026-10-03' <= t <= '2026-10-12T16:00':
rows.setdefault(t, {})[r['sensor_name']] = r
for t in sorted(rows):
d = rows[t]; a, b = d['REF42'], d['REFG43N']
gx = float(a['x']) - float(b['x']); gy = float(a['y']) - float(b['y'])
tg = [s for s in d.values() if s['pattern_type'] not in ('global_ref',) and s['corr_x']]
cx = st.median(float(s['corr_x']) for s in tg); cy = st.median(float(s['corr_y']) for s in tg)
print(t, f"gap_x {gx:9.3f} gap_y {gy:8.3f} med_corr_x {cx:+.3f} med_corr_y {cy:+.3f}")
@@ -0,0 +1,16 @@
"""Per target, median of a corr column in windows (from the summary table at native step), and the difference in noises."""
import sys, statistics as st
f, wins = sys.argv[1], [w.split('/') for w in sys.argv[2:]]
rows, hdr = [], None
for line in open(f):
p = line.split()
if p and p[0] == 'start': hdr = p[1:]; continue
if hdr and p and p[0].startswith('2026-'): rows.append((p[0], p[1:]))
def val(s):
try: return float(s.rstrip('*o'))
except ValueError: return None
for i, s in enumerate(hdr):
serie = [(t, val(v[i])) for t, v in rows if val(v[i]) is not None]
noise = st.median(abs(b[1]-a[1]) for a, b in zip(serie, serie[1:]))
meds = [st.median([x for t, x in serie if a <= t[:16]+'Z' <= b] or [float('nan')]) for a, b in wins]
print(f"{s:11s} noise {noise:.4f} " + ' '.join(f"{m:+.4f}" for m in meds) + ' d(noises): ' + ' '.join(f"{(m-meds[0])/noise:+.1f}" for m in meds[1:]))
@@ -0,0 +1,17 @@
"""Write labels.jsonl: MICHIGAN-1 kept verbatim (from work/existing.jsonl, without id), MICHIGAN-2 and -4 corrected, one added."""
import json
O = 'labeling/auto/analyses/MICHIGAN_2026-10-03T0000_2026-10-12T1600'
old = {d['id']: d for d in map(json.loads, open(f'{O}/work/existing.jsonl'))}
m1 = {k: v for k, v in old['MICHIGAN-1'].items() if k != 'id'}
expert = old['MICHIGAN-4']['description'].split('According to the expert, ', 1)[1]
labels = [
m1,
{"start": "2026-10-04T20:00Z", "end": "2026-10-05T12:00Z", "family": "Thermal displacement",
"description": "On 5 October, at the 00:00 measurement, the corr_y of all the targets rises together, a little, most clearly for COLH19N', POUH19N', COLH21'', COLH18S''' and COLH19N''; it partly comes back at 04:00, when the corr_x of several targets (COLH20'', POUH21'', COLH21' and REFH22') rises a little in turn; the corr_y of COLH19N', POUH19N' and COLH19N'' rises a little again at 08:00, then everything is back at 12:00. At the same time, the camera housing temperature drops at once and stays low until it rises again at 12:00, and the camera moves with it: at 00:00 its rotation turns and its tilt goes back a little, at 04:00 and 08:00 it nods forward and its scale drops, at 12:00 it comes back. The raw positions of all the sensors, references included, slide together, the targets without moving relative to each other; the reference REFPOU22 slides a little differently from the targets, and the compensation, still on three references, does not erase all of this movement. COLH21' and REFH22', still placed on the same point (labelled apart), move like the others. Same signature as the night of 6 October (labelled apart). Cause deduced from the curves, the camera pose and the camera housing temperature; nothing in front of the targets on the 00:00 and 08:00 images. Small deviation. For the customer: this is not a movement of the structure, nothing to do."},
{"start": "2026-10-06T00:00Z", "end": "2026-10-06T20:00Z", "family": "Thermal displacement",
"description": "On 6 October, at the 04:00 measurement, the corr_y of most targets goes down together, a little, COLH20', COLH21', REFH22', COLH21'', POUH21'' and COLH18S''' the most, and the corr_x of COLH21' and COLH20' rises a little; most come back at 08:00, but the corr_y of COLH18S''' stays low, and a little that of POUH18' and COLH18', until 16:00, and comes back at 20:00. At the same moment, the camera housing temperature makes a brief drop, lowest at 04:00, then rises again until 16:00; the camera nods forward (tilt rising, scale dropping) at 04:00, then comes back gradually until 16:00, and the raw positions of all the sensors, references included, slide together: it is the camera that moves, not the targets; the compensation, still on three references, does not erase all of this movement. The error_value of COLH18', POUH18' and COLH19N' rises a little at 04:00 and returns at 16:00. Same signature as the night of 5 October (labelled apart). Cause deduced from the curves, the camera pose and the camera housing temperature; nothing in front of the targets on the 04:00 image. Small deviation. For the customer: this is not a movement of the structure, nothing to do."},
{"start": "2026-10-08T00:00Z", "end": "2026-10-12T16:00Z", "family": "Displacement",
"description": "From 8 to 12 October, the camera gradually nods forward: its tilt rises, its pan and its rotation decrease, and all the targets slide together in the image, shrinking slightly, without moving relative to each other. Until 9 October, on three references, the compensation absorbs most of this movement, but not all: from the 04:00 measurement on 8 October, when the camera takes a clearer step forward, the corr_x of COLH18', POUH18', POUH19N', COLH21', COLH19N' and COLH18S''' rises a little and stays up, the corr_y of POUH19N' goes down a little, and the corrections of the references REF42 and REFG43N move apart. Since the loss of REFPOU22 on 9 October at 12:00 (separate label, which also covers the jump of that measurement), the compensation runs on REF42 and REFG43N alone, which do not follow this movement as the targets do (the scale computed on them even rises): the corr_x of all the targets rises slowly and clearly until the end of the range, under the back-and-forth agitation of the two-reference compensation. Its stop and slight return from the evening of 12 October are labelled apart. Deduced from the curves, the camera pose and the raw positions of the targets; nothing on the images. According to the expert, " + expert},
]
with open(f'{O}/labels.jsonl', 'w') as f:
for d in labels: f.write(json.dumps(d, ensure_ascii=False) + '\n')