auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df979301b5
commit
2a5d87659b
279 files changed
+10758
-710
No files matched your search
@@ -0,0 +1,19 @@
|
||||
"""Figure: camera tilt and corr_y of the targets that follow it, 2h medians, from tilt_vs_targets.py output."""
|
||||
import sys, matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
|
||||
from datetime import datetime
|
||||
rows=[l.split() for l in open(sys.argv[1]) if l[:2]=='10' and '-' in l[:6]]
|
||||
t=[datetime.strptime('2026-'+r[0],'%Y-%m-%dT%H:%M') for r in rows]
|
||||
def col(i): return [float(r[i]) if r[i] != '-' else float('nan') for r in rows]
|
||||
fig,ax=plt.subplots(2,1,figsize=(11,6),sharex=True)
|
||||
ax[0].plot(t,col(1),'k.-'); ax[0].set_ylabel('camera tilt\n(thousandths of a degree)')
|
||||
names=['D-302-B','D-303-B','D-304-B','P-301-B','D-306-B']
|
||||
for j,n in enumerate(names):
|
||||
v=col(3+j); m=sorted(x for x in v if x==x)[len(v)//2]
|
||||
ax[1].plot(t,[x-m for x in v],'.-',label=n)
|
||||
ax[1].set_ylabel('corrected vertical curve\nminus its median (px)'); ax[1].legend(ncol=5,fontsize=8)
|
||||
for a in ax:
|
||||
for s,e in [('2026-10-11T14:00','2026-10-13T02:00'),('2026-10-17T16:00','2026-10-20T00:30')]:
|
||||
a.axvspan(datetime.fromisoformat(s),datetime.fromisoformat(e),color='0.9')
|
||||
a.grid(alpha=.3)
|
||||
fig.suptitle('CLARIDEN: the camera tilt and the targets that follow it (2 h medians, UTC)')
|
||||
fig.tight_layout(); fig.savefig(sys.argv[2],dpi=90); print(sys.argv[2])
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Instants where many targets deviate together, same sign, from their rolling local level.
|
||||
For each target and column (corr_x, corr_y), deviation = value - median of the +-N neighbouring
|
||||
accepted measurements (itself excluded), in units of the target's own noise (median |step|).
|
||||
Prints the instants where at least K targets exceed THR noises with the same sign.
|
||||
Usage: python3 group_flips.py START END [N=6] [THR=3] [K=6]"""
|
||||
import csv, glob, sys, statistics as st
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
S = datetime.fromisoformat(sys.argv[1].replace('Z', '+00:00'))
|
||||
E = datetime.fromisoformat(sys.argv[2].replace('Z', '+00:00'))
|
||||
N = int(sys.argv[3]) if len(sys.argv) > 3 else 6
|
||||
THR = float(sys.argv[4]) if len(sys.argv) > 4 else 3
|
||||
K = int(sys.argv[5]) if len(sys.argv) > 5 else 6
|
||||
REFS = {'D-302-C', 'PR-291', 'PR-293', 'PR-295', 'PR-296'}
|
||||
data = {}
|
||||
d = S.date() - timedelta(days=1)
|
||||
while d <= E.date():
|
||||
for f in glob.glob(f'data/CLARIDEN/displacement/CLARIDEN_advanced_{d:%Y%m%d}.csv'):
|
||||
for r in csv.DictReader(open(f)):
|
||||
t = r['time_utc'][:16].replace(' ', 'T') + 'Z'
|
||||
tt = datetime.fromisoformat(t.replace('Z', '+00:00'))
|
||||
if not (S <= tt <= E) or r['sensor_name'] in REFS or r['corr_x'] == '':
|
||||
continue
|
||||
data.setdefault(r['sensor_name'], {})[t] = (float(r['corr_x']), float(r['corr_y']))
|
||||
d += timedelta(days=1)
|
||||
dev = {}
|
||||
for s, series in data.items():
|
||||
ts = sorted(series)
|
||||
for i, col in enumerate(('x', 'y')):
|
||||
v = [series[t][i] for t in ts]
|
||||
noise = st.median(abs(v[j + 1] - v[j]) for j in range(len(v) - 1)) or 1e-9
|
||||
for j, t in enumerate(ts):
|
||||
nb = v[max(0, j - N):j] + v[j + 1:j + 1 + N]
|
||||
dev.setdefault(t, []).append((s, col, (v[j] - st.median(nb)) / noise))
|
||||
for t in sorted(dev):
|
||||
for col in ('x', 'y'):
|
||||
for sign in (1, -1):
|
||||
hit = [(s, round(z, 1)) for s, c, z in dev[t] if c == col and z * sign > THR]
|
||||
if len(hit) >= K:
|
||||
print(t, f'corr_{col}', '+' if sign > 0 else '-', len(hit), ' '.join(f'{s}:{z}' for s, z in sorted(hit, key=lambda h: -abs(h[1]))))
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Per-step medians of camera tilt, housing temperature and corr_y/corr_x of chosen targets; and
|
||||
the regression of each target's corr on the tilt over chosen windows.
|
||||
Usage: python3 tilt_vs_targets.py START END STEP_HOURS"""
|
||||
import csv, glob, sys, statistics as st
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
S = datetime.fromisoformat(sys.argv[1].replace('Z', '+00:00'))
|
||||
E = datetime.fromisoformat(sys.argv[2].replace('Z', '+00:00'))
|
||||
H = float(sys.argv[3])
|
||||
T = ['D-302-B', 'D-303-B', 'D-304-B', 'P-301-B', 'D-306-B', 'D-303-C', 'D-304-C', 'D-301-C']
|
||||
base = 'data/CLARIDEN'
|
||||
|
||||
def t(s):
|
||||
return datetime.fromisoformat(s.split('.')[0].replace(' ', 'T')[:19] + '+00:00')
|
||||
|
||||
def days():
|
||||
d = S.date() - timedelta(days=1)
|
||||
while d <= E.date():
|
||||
yield d
|
||||
d += timedelta(days=1)
|
||||
|
||||
corr = {} # (sensor, col) -> list of (time, value)
|
||||
for d in days():
|
||||
for f in glob.glob(f'{base}/displacement/CLARIDEN_advanced_{d:%Y%m%d}.csv'):
|
||||
for r in csv.DictReader(open(f)):
|
||||
tt = t(r['time_utc'])
|
||||
if not (S <= tt <= E) or r['sensor_name'] not in T or r['corr_y'] == '':
|
||||
continue
|
||||
for c in ('corr_x', 'corr_y'):
|
||||
corr.setdefault((r['sensor_name'], c), []).append((tt, float(r[c])))
|
||||
tilt = []
|
||||
for d in days():
|
||||
for f in glob.glob(f'{base}/camera/CLARIDEN_camera_{d:%Y%m%d}.csv'):
|
||||
for r in csv.DictReader(open(f)):
|
||||
tt = t(r['time_utc'])
|
||||
if S <= tt <= E:
|
||||
tilt.append((tt, float(r['median_tilt'])))
|
||||
temp = []
|
||||
for d in days():
|
||||
for f in glob.glob(f'{base}/metrics/CLARIDEN_sensor_GPU_deg_c_{d:%Y_%m_%d}_*.csv'):
|
||||
for r in csv.DictReader(open(f)):
|
||||
tt = t(r['time_utc'])
|
||||
if S <= tt <= E and r['metric'] != '':
|
||||
temp.append((tt, float(r['metric'])))
|
||||
|
||||
def binned(series):
|
||||
out = {}
|
||||
for tt, v in series:
|
||||
k = int((tt - S).total_seconds() // (H * 3600))
|
||||
out.setdefault(k, []).append(v)
|
||||
return {k: st.median(v) for k, v in out.items()}
|
||||
|
||||
bt, bT = binned(tilt), binned(temp)
|
||||
bc = {k: binned(v) for k, v in corr.items()}
|
||||
keys = sorted(bt)
|
||||
print('step_start tilt*1e3 temp ' + ' '.join(f'{s[:7]:>8}y' for s in T) + ' D-302-Bx D-301-Cx')
|
||||
for k in keys:
|
||||
row = [f'{(S + timedelta(hours=k * H)):%m-%dT%H:%M}', f'{bt[k]*1e3:8.2f}', f'{bT.get(k, float("nan")):5.2f}']
|
||||
for s in T:
|
||||
v = bc.get((s, 'corr_y'), {}).get(k)
|
||||
row.append(f'{v:9.3f}' if v is not None else ' -')
|
||||
for s in ('D-302-B', 'D-301-C'):
|
||||
v = bc.get((s, 'corr_x'), {}).get(k)
|
||||
row.append(f'{v:8.3f}' if v is not None else ' -')
|
||||
print(' '.join(row))
|
||||
|
||||
# regression of each target's corr_y on tilt, per measurement
|
||||
tl = dict(tilt)
|
||||
print('\nregression of corr_y on tilt (per measurement): slope px per 1e-3 deg, r')
|
||||
for s in T:
|
||||
pts = [(tl[tt], v) for tt, v in corr.get((s, 'corr_y'), []) if tt in tl]
|
||||
if len(pts) < 10:
|
||||
continue
|
||||
xs, ys = zip(*pts)
|
||||
mx, my = st.mean(xs), st.mean(ys)
|
||||
sxy = sum((a - mx) * (b - my) for a, b in pts); sxx = sum((a - mx) ** 2 for a in xs); syy = sum((b - my) ** 2 for b in ys)
|
||||
print(f'{s:8} slope {sxy / sxx * 1e-3:+.4f} r {sxy / (sxx * syy) ** 0.5:+.2f} n {len(pts)}')
|
||||
Reference in new issue
Block a user