.
This commit is contained in:
1 parent
8e4ebca342
commit
58b630f2d5
112 files changed
+3172
-295
No files matched your search
@@ -0,0 +1,31 @@
|
||||
"""Distribution of the common deviation: per instant, the median over the targets of the corr deviation (rolling +-N
|
||||
measurements, in own noises) and the number of targets beyond K noises; lists the instants beyond K on the median.
|
||||
usage: common_dist.py FOLDER START END COL [N] [K]"""
|
||||
import sys, statistics as st
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
folder, start, end, col = sys.argv[1:5]
|
||||
N = int(sys.argv[5]) if len(sys.argv) > 5 else 8
|
||||
K = float(sys.argv[6]) if len(sys.argv) > 6 else 5
|
||||
D = data.load(folder); S, E = data.parse_time(start), data.parse_time(end)
|
||||
tg = [s for s in D if not data.is_reference(D[s])]
|
||||
dev = {}
|
||||
for s in tg:
|
||||
vals = [(m["t"], m[col]) for m in D[s] if S <= m["t"] <= E]
|
||||
v = [x for _, x in vals if x is not None]
|
||||
noise = st.median(abs(a - b) for a, b in zip(v[1:], v[:-1]))
|
||||
for i, (t, x) in enumerate(vals):
|
||||
if x is None: continue
|
||||
nb = [y for _, y in vals[max(0, i - N):i] + vals[i + 1:i + 1 + N] if y is not None]
|
||||
dev.setdefault(t, {})[s] = (x - st.median(nb)) / noise
|
||||
med = {t: st.median(d.values()) for t, d in dev.items()}
|
||||
a = sorted(abs(x) for x in med.values())
|
||||
n = len(a)
|
||||
print(col, "instants", n, "|median dev| percentiles 50/90/99/99.9:", [round(a[int(p * (n - 1))], 1) for p in (.5, .9, .99, .999)])
|
||||
for k in (3, 4, 5, 6, 8, 10):
|
||||
print(f" instants with |median dev| > {k}: {sum(x > k for x in a)}; with >=1 target beyond {k}: {sum(any(abs(v) > k for v in d.values()) for d in dev.values())}")
|
||||
print("instants with |median| >", K)
|
||||
for t in sorted(med):
|
||||
if abs(med[t]) > K:
|
||||
d = dev[t]; mx = max(d, key=lambda s: abs(d[s]))
|
||||
print(" ", data.iso(t), f"median {med[t]:+.1f}", f"n>{K}: {sum(abs(v) > K for v in d.values())}", f"max {mx} {d[mx]:+.1f}")
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Per instant, each target's corr_x / corr_y deviation from a rolling median level (+-N measurements, itself excluded),
|
||||
in its own noise (median |step| over the window), plus raw x, y medians and error_value of the refs.
|
||||
usage: devtable.py FOLDER START END [N] [COL] [SHOW_START SHOW_END]"""
|
||||
import sys, statistics as st
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
folder, start, end = sys.argv[1:4]
|
||||
N = int(sys.argv[4]) if len(sys.argv) > 4 else 8
|
||||
col = sys.argv[5] if len(sys.argv) > 5 else "corr_y"
|
||||
s0 = data.parse_time(sys.argv[6]) if len(sys.argv) > 6 else None
|
||||
s1 = data.parse_time(sys.argv[7]) if len(sys.argv) > 7 else None
|
||||
D = data.load(folder)
|
||||
S, E = data.parse_time(start), data.parse_time(end)
|
||||
tg = [s for s in D if not data.is_reference(D[s])]
|
||||
refs = [s for s in D if data.is_reference(D[s])]
|
||||
series = {}
|
||||
for s in tg:
|
||||
ms = [m for m in D[s] if S <= m["t"] <= E]
|
||||
vals = [(m["t"], m[col]) for m in ms]
|
||||
v = [x for _, x in vals if x is not None]
|
||||
noise = st.median(abs(a - b) for a, b in zip(v[1:], v[:-1]))
|
||||
dev = {}
|
||||
for i, (t, x) in enumerate(vals):
|
||||
if x is None: dev[t] = None; continue
|
||||
nb = [y for _, y in vals[max(0, i - N):i] + vals[i + 1:i + 1 + N] if y is not None]
|
||||
dev[t] = (x - st.median(nb)) / noise
|
||||
series[s] = dev
|
||||
times = sorted({m["t"] for m in D[tg[0]] if S <= m["t"] <= E})
|
||||
print("time " + " ".join(f"{s:>5}" for s in tg) + " | ev " + " ".join(refs))
|
||||
for t in times:
|
||||
if s0 and (t < s0 or t > s1): continue
|
||||
row = []
|
||||
for s in tg:
|
||||
d = series[s].get(t)
|
||||
row.append(" rej" if d is None else f"{d:5.1f}")
|
||||
ev = []
|
||||
for r in refs:
|
||||
m = next((m for m in D[r] if m["t"] == t), None)
|
||||
ev.append(f"{m['error_value']:.3f}" if m else "-")
|
||||
print(data.iso(t), " ".join(row), " |", " ".join(ev))
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Hourly median of corr_x_mm, corr_y_mm per target (minus its median over the window) against the housing temperature
|
||||
and the pose scale. usage: slowfig.py FOLDER START END OUT.png [SENSORS]"""
|
||||
import sys, statistics as st
|
||||
from collections import defaultdict
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
|
||||
folder, start, end, out = sys.argv[1:5]
|
||||
S, E = data.parse_time(start), data.parse_time(end)
|
||||
D = data.load(folder)
|
||||
sens = sys.argv[5].split(",") if len(sys.argv) > 5 else [s for s in D if not data.is_reference(D[s])]
|
||||
fig, ax = plt.subplots(4, 1, figsize=(14, 12), sharex=True)
|
||||
for s in sens:
|
||||
for k, col in enumerate(("corr_x_mm", "corr_y_mm")):
|
||||
h = defaultdict(list)
|
||||
for m in D[s]:
|
||||
if S <= m["t"] <= E and m[col] is not None:
|
||||
h[m["t"].replace(minute=0, second=0, microsecond=0)].append(m[col])
|
||||
ts = sorted(h); v = [st.median(h[t]) for t in ts]; md = st.median(v)
|
||||
ax[k].plot(ts, [x - md for x in v], label=s, lw=1)
|
||||
for k, c in enumerate(("corr_x_mm hourly median - level", "corr_y_mm hourly median - level")):
|
||||
ax[k].set_ylabel(c); ax[k].grid(alpha=.3)
|
||||
ax[0].legend(ncol=6, fontsize=7)
|
||||
T = [(t, v) for t, v in data.load_temperature(folder) if S <= t <= E]
|
||||
ax[2].plot([t for t, _ in T], [v for _, v in T], "k"); ax[2].set_ylabel("housing temperature"); ax[2].grid(alpha=.3)
|
||||
C = [p for p in data.load_camera(folder) if S <= p["t"] <= E]
|
||||
ax[3].plot([p["t"] for p in C], [p.get("scale") for p in C], lw=.6); ax[3].set_ylabel("cam scale"); ax[3].grid(alpha=.3)
|
||||
fig.tight_layout(); fig.savefig(out, dpi=80); print(out)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""For windows T1/T2, list the instants where targets exceed K noises (rolling +-N level) on corr_x or corr_y:
|
||||
count, median of corr_x and corr_y deviations, and the sensors. usage: touched.py FOLDER START END T1/T2 [T1/T2 ...]"""
|
||||
import sys, statistics as st
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
folder, start, end = sys.argv[1:4]; wins = sys.argv[4:]; N, K = 8, 5
|
||||
D = data.load(folder); S, E = data.parse_time(start), data.parse_time(end)
|
||||
tg = [s for s in D if not data.is_reference(D[s])]
|
||||
dev = {}
|
||||
for col in ("corr_x", "corr_y"):
|
||||
for s in tg:
|
||||
vals = [(m["t"], m[col]) for m in D[s] if S <= m["t"] <= E]
|
||||
v = [x for _, x in vals if x is not None]
|
||||
noise = st.median(abs(a - b) for a, b in zip(v[1:], v[:-1]))
|
||||
for i, (t, x) in enumerate(vals):
|
||||
if x is None: continue
|
||||
nb = [y for _, y in vals[max(0, i - N):i] + vals[i + 1:i + 1 + N] if y is not None]
|
||||
dev.setdefault(t, {}).setdefault(col, {})[s] = (x - st.median(nb)) / noise
|
||||
for w in wins:
|
||||
a, b = map(data.parse_time, w.split("/"))
|
||||
print("==", w)
|
||||
for t in sorted(dev):
|
||||
if not a <= t <= b: continue
|
||||
d = dev[t]; out = []
|
||||
for col in ("corr_x", "corr_y"):
|
||||
out += [f"{s}.{col[-1]}{d[col][s]:+.0f}" for s in tg if s in d.get(col, {}) and abs(d[col][s]) > K]
|
||||
mx = st.median(d["corr_x"].values()); my = st.median(d["corr_y"].values())
|
||||
print(" ", data.iso(t), f"med x {mx:+.1f} y {my:+.1f}", len(out), " ".join(out))
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Summarise the hourly weather of a station over a range: precipitation, temperature, humidity, wind, sun, snow."""
|
||||
import csv, glob, sys
|
||||
folder, start, end = sys.argv[1:4]
|
||||
rows = []
|
||||
for f in sorted(glob.glob(f"{folder}/weather/*.csv")):
|
||||
for r in csv.DictReader(open(f)):
|
||||
if start <= r["time_utc"] <= end:
|
||||
rows.append(r)
|
||||
print("station:", rows[0]["station_name"], "distance_km:", rows[0]["distance_km"], "height_m:", rows[0]["height_m"])
|
||||
print("time prec T hum wind gust sun rad snow")
|
||||
for r in rows:
|
||||
print(r["time_utc"], r["precipitation_mm"].rjust(4), r["temperature_c"].rjust(5), r["humidity_pct"].rjust(5),
|
||||
r["wind_speed_kmh"].rjust(5), r["wind_gust_kmh"].rjust(5), r["sunshine_min"].rjust(3), r["global_radiation_wm2"].rjust(4), r["snow_depth_cm"])
|
||||
Reference in new issue
Block a user