auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 15:13:44 +01:00
1 parent df979301b5
commit 2a5d87659b
279 files changed
+10758 -710

No files matched your search

@@ -0,0 +1,19 @@
"""Figure: camera tilt and corr_y of the targets that follow it, 2h medians, from tilt_vs_targets.py output."""
import sys, matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
from datetime import datetime
rows=[l.split() for l in open(sys.argv[1]) if l[:2]=='10' and '-' in l[:6]]
t=[datetime.strptime('2026-'+r[0],'%Y-%m-%dT%H:%M') for r in rows]
def col(i): return [float(r[i]) if r[i] != '-' else float('nan') for r in rows]
fig,ax=plt.subplots(2,1,figsize=(11,6),sharex=True)
ax[0].plot(t,col(1),'k.-'); ax[0].set_ylabel('camera tilt\n(thousandths of a degree)')
names=['D-302-B','D-303-B','D-304-B','P-301-B','D-306-B']
for j,n in enumerate(names):
v=col(3+j); m=sorted(x for x in v if x==x)[len(v)//2]
ax[1].plot(t,[x-m for x in v],'.-',label=n)
ax[1].set_ylabel('corrected vertical curve\nminus its median (px)'); ax[1].legend(ncol=5,fontsize=8)
for a in ax:
for s,e in [('2026-10-11T14:00','2026-10-13T02:00'),('2026-10-17T16:00','2026-10-20T00:30')]:
a.axvspan(datetime.fromisoformat(s),datetime.fromisoformat(e),color='0.9')
a.grid(alpha=.3)
fig.suptitle('CLARIDEN: the camera tilt and the targets that follow it (2 h medians, UTC)')
fig.tight_layout(); fig.savefig(sys.argv[2],dpi=90); print(sys.argv[2])
@@ -0,0 +1,40 @@
"""Instants where many targets deviate together, same sign, from their rolling local level.
For each target and column (corr_x, corr_y), deviation = value - median of the +-N neighbouring
accepted measurements (itself excluded), in units of the target's own noise (median |step|).
Prints the instants where at least K targets exceed THR noises with the same sign.
Usage: python3 group_flips.py START END [N=6] [THR=3] [K=6]"""
import csv, glob, sys, statistics as st
from datetime import datetime, timedelta
S = datetime.fromisoformat(sys.argv[1].replace('Z', '+00:00'))
E = datetime.fromisoformat(sys.argv[2].replace('Z', '+00:00'))
N = int(sys.argv[3]) if len(sys.argv) > 3 else 6
THR = float(sys.argv[4]) if len(sys.argv) > 4 else 3
K = int(sys.argv[5]) if len(sys.argv) > 5 else 6
REFS = {'D-302-C', 'PR-291', 'PR-293', 'PR-295', 'PR-296'}
data = {}
d = S.date() - timedelta(days=1)
while d <= E.date():
for f in glob.glob(f'data/CLARIDEN/displacement/CLARIDEN_advanced_{d:%Y%m%d}.csv'):
for r in csv.DictReader(open(f)):
t = r['time_utc'][:16].replace(' ', 'T') + 'Z'
tt = datetime.fromisoformat(t.replace('Z', '+00:00'))
if not (S <= tt <= E) or r['sensor_name'] in REFS or r['corr_x'] == '':
continue
data.setdefault(r['sensor_name'], {})[t] = (float(r['corr_x']), float(r['corr_y']))
d += timedelta(days=1)
dev = {}
for s, series in data.items():
ts = sorted(series)
for i, col in enumerate(('x', 'y')):
v = [series[t][i] for t in ts]
noise = st.median(abs(v[j + 1] - v[j]) for j in range(len(v) - 1)) or 1e-9
for j, t in enumerate(ts):
nb = v[max(0, j - N):j] + v[j + 1:j + 1 + N]
dev.setdefault(t, []).append((s, col, (v[j] - st.median(nb)) / noise))
for t in sorted(dev):
for col in ('x', 'y'):
for sign in (1, -1):
hit = [(s, round(z, 1)) for s, c, z in dev[t] if c == col and z * sign > THR]
if len(hit) >= K:
print(t, f'corr_{col}', '+' if sign > 0 else '-', len(hit), ' '.join(f'{s}:{z}' for s, z in sorted(hit, key=lambda h: -abs(h[1]))))
@@ -0,0 +1,77 @@
"""Per-step medians of camera tilt, housing temperature and corr_y/corr_x of chosen targets; and
the regression of each target's corr on the tilt over chosen windows.
Usage: python3 tilt_vs_targets.py START END STEP_HOURS"""
import csv, glob, sys, statistics as st
from datetime import datetime, timedelta, timezone
S = datetime.fromisoformat(sys.argv[1].replace('Z', '+00:00'))
E = datetime.fromisoformat(sys.argv[2].replace('Z', '+00:00'))
H = float(sys.argv[3])
T = ['D-302-B', 'D-303-B', 'D-304-B', 'P-301-B', 'D-306-B', 'D-303-C', 'D-304-C', 'D-301-C']
base = 'data/CLARIDEN'
def t(s):
return datetime.fromisoformat(s.split('.')[0].replace(' ', 'T')[:19] + '+00:00')
def days():
d = S.date() - timedelta(days=1)
while d <= E.date():
yield d
d += timedelta(days=1)
corr = {} # (sensor, col) -> list of (time, value)
for d in days():
for f in glob.glob(f'{base}/displacement/CLARIDEN_advanced_{d:%Y%m%d}.csv'):
for r in csv.DictReader(open(f)):
tt = t(r['time_utc'])
if not (S <= tt <= E) or r['sensor_name'] not in T or r['corr_y'] == '':
continue
for c in ('corr_x', 'corr_y'):
corr.setdefault((r['sensor_name'], c), []).append((tt, float(r[c])))
tilt = []
for d in days():
for f in glob.glob(f'{base}/camera/CLARIDEN_camera_{d:%Y%m%d}.csv'):
for r in csv.DictReader(open(f)):
tt = t(r['time_utc'])
if S <= tt <= E:
tilt.append((tt, float(r['median_tilt'])))
temp = []
for d in days():
for f in glob.glob(f'{base}/metrics/CLARIDEN_sensor_GPU_deg_c_{d:%Y_%m_%d}_*.csv'):
for r in csv.DictReader(open(f)):
tt = t(r['time_utc'])
if S <= tt <= E and r['metric'] != '':
temp.append((tt, float(r['metric'])))
def binned(series):
out = {}
for tt, v in series:
k = int((tt - S).total_seconds() // (H * 3600))
out.setdefault(k, []).append(v)
return {k: st.median(v) for k, v in out.items()}
bt, bT = binned(tilt), binned(temp)
bc = {k: binned(v) for k, v in corr.items()}
keys = sorted(bt)
print('step_start tilt*1e3 temp ' + ' '.join(f'{s[:7]:>8}y' for s in T) + ' D-302-Bx D-301-Cx')
for k in keys:
row = [f'{(S + timedelta(hours=k * H)):%m-%dT%H:%M}', f'{bt[k]*1e3:8.2f}', f'{bT.get(k, float("nan")):5.2f}']
for s in T:
v = bc.get((s, 'corr_y'), {}).get(k)
row.append(f'{v:9.3f}' if v is not None else ' -')
for s in ('D-302-B', 'D-301-C'):
v = bc.get((s, 'corr_x'), {}).get(k)
row.append(f'{v:8.3f}' if v is not None else ' -')
print(' '.join(row))
# regression of each target's corr_y on tilt, per measurement
tl = dict(tilt)
print('\nregression of corr_y on tilt (per measurement): slope px per 1e-3 deg, r')
for s in T:
pts = [(tl[tt], v) for tt, v in corr.get((s, 'corr_y'), []) if tt in tl]
if len(pts) < 10:
continue
xs, ys = zip(*pts)
mx, my = st.mean(xs), st.mean(ys)
sxy = sum((a - mx) * (b - my) for a, b in pts); sxx = sum((a - mx) ** 2 for a in xs); syy = sum((b - my) ** 2 for b in ys)
print(f'{s:8} slope {sxy / sxx * 1e-3:+.4f} r {sxy / (sxx * syy) ** 0.5:+.2f} n {len(pts)}')