This commit is contained in:
maxime committed 2026-11-18 10:51:31 +01:00
1 parent 8e4ebca342
commit 58b630f2d5
112 files changed
+3172 -295

No files matched your search

@@ -0,0 +1,31 @@
"""Distribution of the common deviation: per instant, the median over the targets of the corr deviation (rolling +-N
measurements, in own noises) and the number of targets beyond K noises; lists the instants beyond K on the median.
usage: common_dist.py FOLDER START END COL [N] [K]"""
import sys, statistics as st
sys.path.insert(0, "tools/scripts")
import data
folder, start, end, col = sys.argv[1:5]
N = int(sys.argv[5]) if len(sys.argv) > 5 else 8
K = float(sys.argv[6]) if len(sys.argv) > 6 else 5
D = data.load(folder); S, E = data.parse_time(start), data.parse_time(end)
tg = [s for s in D if not data.is_reference(D[s])]
dev = {}
for s in tg:
vals = [(m["t"], m[col]) for m in D[s] if S <= m["t"] <= E]
v = [x for _, x in vals if x is not None]
noise = st.median(abs(a - b) for a, b in zip(v[1:], v[:-1]))
for i, (t, x) in enumerate(vals):
if x is None: continue
nb = [y for _, y in vals[max(0, i - N):i] + vals[i + 1:i + 1 + N] if y is not None]
dev.setdefault(t, {})[s] = (x - st.median(nb)) / noise
med = {t: st.median(d.values()) for t, d in dev.items()}
a = sorted(abs(x) for x in med.values())
n = len(a)
print(col, "instants", n, "|median dev| percentiles 50/90/99/99.9:", [round(a[int(p * (n - 1))], 1) for p in (.5, .9, .99, .999)])
for k in (3, 4, 5, 6, 8, 10):
print(f" instants with |median dev| > {k}: {sum(x > k for x in a)}; with >=1 target beyond {k}: {sum(any(abs(v) > k for v in d.values()) for d in dev.values())}")
print("instants with |median| >", K)
for t in sorted(med):
if abs(med[t]) > K:
d = dev[t]; mx = max(d, key=lambda s: abs(d[s]))
print(" ", data.iso(t), f"median {med[t]:+.1f}", f"n>{K}: {sum(abs(v) > K for v in d.values())}", f"max {mx} {d[mx]:+.1f}")
@@ -0,0 +1,40 @@
"""Per instant, each target's corr_x / corr_y deviation from a rolling median level (+-N measurements, itself excluded),
in its own noise (median |step| over the window), plus raw x, y medians and error_value of the refs.
usage: devtable.py FOLDER START END [N] [COL] [SHOW_START SHOW_END]"""
import sys, statistics as st
sys.path.insert(0, "tools/scripts")
import data
folder, start, end = sys.argv[1:4]
N = int(sys.argv[4]) if len(sys.argv) > 4 else 8
col = sys.argv[5] if len(sys.argv) > 5 else "corr_y"
s0 = data.parse_time(sys.argv[6]) if len(sys.argv) > 6 else None
s1 = data.parse_time(sys.argv[7]) if len(sys.argv) > 7 else None
D = data.load(folder)
S, E = data.parse_time(start), data.parse_time(end)
tg = [s for s in D if not data.is_reference(D[s])]
refs = [s for s in D if data.is_reference(D[s])]
series = {}
for s in tg:
ms = [m for m in D[s] if S <= m["t"] <= E]
vals = [(m["t"], m[col]) for m in ms]
v = [x for _, x in vals if x is not None]
noise = st.median(abs(a - b) for a, b in zip(v[1:], v[:-1]))
dev = {}
for i, (t, x) in enumerate(vals):
if x is None: dev[t] = None; continue
nb = [y for _, y in vals[max(0, i - N):i] + vals[i + 1:i + 1 + N] if y is not None]
dev[t] = (x - st.median(nb)) / noise
series[s] = dev
times = sorted({m["t"] for m in D[tg[0]] if S <= m["t"] <= E})
print("time " + " ".join(f"{s:>5}" for s in tg) + " | ev " + " ".join(refs))
for t in times:
if s0 and (t < s0 or t > s1): continue
row = []
for s in tg:
d = series[s].get(t)
row.append(" rej" if d is None else f"{d:5.1f}")
ev = []
for r in refs:
m = next((m for m in D[r] if m["t"] == t), None)
ev.append(f"{m['error_value']:.3f}" if m else "-")
print(data.iso(t), " ".join(row), " |", " ".join(ev))
@@ -0,0 +1,28 @@
"""Hourly median of corr_x_mm, corr_y_mm per target (minus its median over the window) against the housing temperature
and the pose scale. usage: slowfig.py FOLDER START END OUT.png [SENSORS]"""
import sys, statistics as st
from collections import defaultdict
sys.path.insert(0, "tools/scripts")
import data
import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
folder, start, end, out = sys.argv[1:5]
S, E = data.parse_time(start), data.parse_time(end)
D = data.load(folder)
sens = sys.argv[5].split(",") if len(sys.argv) > 5 else [s for s in D if not data.is_reference(D[s])]
fig, ax = plt.subplots(4, 1, figsize=(14, 12), sharex=True)
for s in sens:
for k, col in enumerate(("corr_x_mm", "corr_y_mm")):
h = defaultdict(list)
for m in D[s]:
if S <= m["t"] <= E and m[col] is not None:
h[m["t"].replace(minute=0, second=0, microsecond=0)].append(m[col])
ts = sorted(h); v = [st.median(h[t]) for t in ts]; md = st.median(v)
ax[k].plot(ts, [x - md for x in v], label=s, lw=1)
for k, c in enumerate(("corr_x_mm hourly median - level", "corr_y_mm hourly median - level")):
ax[k].set_ylabel(c); ax[k].grid(alpha=.3)
ax[0].legend(ncol=6, fontsize=7)
T = [(t, v) for t, v in data.load_temperature(folder) if S <= t <= E]
ax[2].plot([t for t, _ in T], [v for _, v in T], "k"); ax[2].set_ylabel("housing temperature"); ax[2].grid(alpha=.3)
C = [p for p in data.load_camera(folder) if S <= p["t"] <= E]
ax[3].plot([p["t"] for p in C], [p.get("scale") for p in C], lw=.6); ax[3].set_ylabel("cam scale"); ax[3].grid(alpha=.3)
fig.tight_layout(); fig.savefig(out, dpi=80); print(out)
@@ -0,0 +1,28 @@
"""For windows T1/T2, list the instants where targets exceed K noises (rolling +-N level) on corr_x or corr_y:
count, median of corr_x and corr_y deviations, and the sensors. usage: touched.py FOLDER START END T1/T2 [T1/T2 ...]"""
import sys, statistics as st
sys.path.insert(0, "tools/scripts")
import data
folder, start, end = sys.argv[1:4]; wins = sys.argv[4:]; N, K = 8, 5
D = data.load(folder); S, E = data.parse_time(start), data.parse_time(end)
tg = [s for s in D if not data.is_reference(D[s])]
dev = {}
for col in ("corr_x", "corr_y"):
for s in tg:
vals = [(m["t"], m[col]) for m in D[s] if S <= m["t"] <= E]
v = [x for _, x in vals if x is not None]
noise = st.median(abs(a - b) for a, b in zip(v[1:], v[:-1]))
for i, (t, x) in enumerate(vals):
if x is None: continue
nb = [y for _, y in vals[max(0, i - N):i] + vals[i + 1:i + 1 + N] if y is not None]
dev.setdefault(t, {}).setdefault(col, {})[s] = (x - st.median(nb)) / noise
for w in wins:
a, b = map(data.parse_time, w.split("/"))
print("==", w)
for t in sorted(dev):
if not a <= t <= b: continue
d = dev[t]; out = []
for col in ("corr_x", "corr_y"):
out += [f"{s}.{col[-1]}{d[col][s]:+.0f}" for s in tg if s in d.get(col, {}) and abs(d[col][s]) > K]
mx = st.median(d["corr_x"].values()); my = st.median(d["corr_y"].values())
print(" ", data.iso(t), f"med x {mx:+.1f} y {my:+.1f}", len(out), " ".join(out))
@@ -0,0 +1,13 @@
"""Summarise the hourly weather of a station over a range: precipitation, temperature, humidity, wind, sun, snow."""
import csv, glob, sys
folder, start, end = sys.argv[1:4]
rows = []
for f in sorted(glob.glob(f"{folder}/weather/*.csv")):
for r in csv.DictReader(open(f)):
if start <= r["time_utc"] <= end:
rows.append(r)
print("station:", rows[0]["station_name"], "distance_km:", rows[0]["distance_km"], "height_m:", rows[0]["height_m"])
print("time prec T hum wind gust sun rad snow")
for r in rows:
print(r["time_utc"], r["precipitation_mm"].rjust(4), r["temperature_c"].rjust(5), r["humidity_pct"].rjust(5),
r["wind_speed_kmh"].rjust(5), r["wind_gust_kmh"].rjust(5), r["sunshine_min"].rjust(3), r["global_radiation_wm2"].rjust(4), r["snow_depth_cm"])