2026-11-16 15:48:51 +01:00
|
|
|
"""The dataset of the pilot: for each labelled station, the measurements of its labelled range turned into arrays.
|
|
|
|
|
|
|
|
|
|
Per instant (a minute where at least one sensor has a measurement, as bench/score.py), per sensor:
|
|
|
|
|
Xs [T, S, Fs]: each sensor against its own usual level (rolling median and IQR over the HISTORY before), on the
|
|
|
|
|
instant and on windows in hours before and after (the step goes from 5 min to 4 h); NaN where it does not exist.
|
|
|
|
|
Per instant, for the station:
|
|
|
|
|
Xc [T, Fc]: what the sensors do together (shares, quantiles, common mode), the references, the camera pose, the
|
|
|
|
|
housing temperature, the time of day, the step.
|
|
|
|
|
Per instant, the families of the labels covering it, with the conventions of bench/score.py (bounds excluded, except a
|
|
|
|
|
hole without a measurement inside):
|
|
|
|
|
Y [T, K], over the FAMILIES.
|
|
|
|
|
|
|
|
|
|
The measurements are read through detectors/scripts/data.py, the reading of the benchmark. One .npz per station in --out.
|
|
|
|
|
Usage: python dataset.py --detectors <detectors> --data <data> --labels <labels folder> --out <folder>
|
|
|
|
|
[--stations A,B] [--from YYYY-MM-DD --to YYYY-MM-DD]
|
|
|
|
|
"""
|
|
|
|
|
import argparse
|
|
|
|
|
import json
|
|
|
|
|
import sys
|
|
|
|
|
from datetime import timedelta
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
import pandas as pd
|
|
|
|
|
|
|
|
|
|
FAMILIES = ["Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"]
|
|
|
|
|
HISTORY = timedelta(days=3) # as the detectors: the usual level of a sensor is read on the 3 days before
|
|
|
|
|
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"] # image quality, recent cameras only
|
|
|
|
|
EPS = 1e-9
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def label_families(labels, instants):
|
|
|
|
|
"""Labels and sorted iso instants -> Y [T, K] uint8, as bench/score.py: a label covers the instants strictly inside
|
|
|
|
|
its bounds; a label with no instant strictly inside (a hole) covers its bounds too."""
|
|
|
|
|
y = np.zeros((len(instants), len(FAMILIES)), np.uint8)
|
|
|
|
|
arr = np.array(instants)
|
|
|
|
|
for e in labels:
|
|
|
|
|
if e.get("family") not in FAMILIES:
|
|
|
|
|
continue
|
|
|
|
|
k = FAMILIES.index(e["family"])
|
|
|
|
|
inside = (arr > e["start"]) & (arr < e["end"])
|
|
|
|
|
y[inside if inside.any() else (arr >= e["start"]) & (arr <= e["end"]), k] = 1
|
|
|
|
|
return y
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def robust_z(df, window="3D"):
|
|
|
|
|
"""Each column against its rolling median and IQR over the window before (current included)."""
|
|
|
|
|
r = df.rolling(window, min_periods=10)
|
|
|
|
|
med, iqr = r.median(), r.quantile(0.75) - r.quantile(0.25)
|
|
|
|
|
return (df - med) / (iqr / 1.349 + EPS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def roll(df, window, how, forward=False):
|
|
|
|
|
"""Time window strictly before each instant, or strictly after it (forward) -> the same frame."""
|
|
|
|
|
if not forward:
|
|
|
|
|
return getattr(df.rolling(window, closed="left", min_periods=1), how)()
|
|
|
|
|
rev = df.iloc[::-1]
|
|
|
|
|
rev.index = df.index[-1] - rev.index # increasing distance to the end
|
|
|
|
|
out = getattr(rev.rolling(window, closed="left", min_periods=1), how)().iloc[::-1]
|
|
|
|
|
out.index = df.index
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def wide(measurements, column):
|
|
|
|
|
"""{sensor: [measurement]} -> DataFrame [minute, sensor] of one column."""
|
|
|
|
|
rows = [(m["t"].replace(second=0, microsecond=0), s, m[column]) for s, v in measurements.items() for m in v]
|
|
|
|
|
df = pd.DataFrame(rows, columns=["t", "s", "v"]).astype({"v": float})
|
|
|
|
|
return df.pivot_table(index="t", columns="s", values="v", aggfunc="last", dropna=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sensor_features(measurements, kinds, grid):
|
|
|
|
|
"""-> (names, Xs [T, S, Fs]) on the grid (a DatetimeIndex, minutes, history included)."""
|
|
|
|
|
sensors = list(measurements)
|
|
|
|
|
get = {c: wide(measurements, c).reindex(index=grid, columns=sensors) for c in
|
|
|
|
|
["error_value", "error_threshold", "corr_x", "corr_y"] + QUALITY}
|
|
|
|
|
ev, thr = get["error_value"], get["error_threshold"]
|
|
|
|
|
present = (ev.notna() | get["corr_x"].notna()).astype(float)
|
|
|
|
|
rej = (ev > thr).astype(float).where(ev.notna() & thr.notna())
|
|
|
|
|
ev_z = robust_z(ev)
|
|
|
|
|
f = {"present": present, "rejected": rej, "ev_z": ev_z, "ev_thr": ev / thr,
|
|
|
|
|
"ev_logmed": np.log((ev + EPS) / (ev.rolling("3D", min_periods=10).median() + EPS))}
|
|
|
|
|
targets = [s for s in sensors if kinds[s] != "global_ref"]
|
|
|
|
|
for axis in "xy":
|
|
|
|
|
c = get["corr_" + axis]
|
|
|
|
|
jump = c.ffill().diff().where(c.notna()) # against the last valid measurement
|
|
|
|
|
noise = jump.abs().rolling("3D", min_periods=10).median() / 0.6745 + EPS # MAD of the steps, in sigma
|
|
|
|
|
dev = (c - c.rolling("3D", min_periods=10).median()) / noise
|
|
|
|
|
common = dev[targets].median(axis=1)
|
|
|
|
|
f[f"d{axis}"] = dev
|
|
|
|
|
f[f"j{axis}"] = jump / noise
|
|
|
|
|
f[f"r{axis}"] = dev.sub(common, axis=0) # what the common mode does not explain
|
|
|
|
|
f[f"step_{axis}_6h"] = roll(dev, "6h", "median", forward=True) - roll(dev, "6h", "median")
|
|
|
|
|
f[f"noise_{axis}_1h"] = roll(jump / noise, "1h", "std")
|
|
|
|
|
for w in ("1h", "6h"):
|
|
|
|
|
f[f"ev_z_b{w}"] = roll(ev_z, w, "mean")
|
|
|
|
|
f[f"ev_z_f{w}"] = roll(ev_z, w, "mean", forward=True)
|
|
|
|
|
f[f"rej_b{w}"] = roll(rej, w, "mean")
|
|
|
|
|
f[f"rej_f{w}"] = roll(rej, w, "mean", forward=True)
|
|
|
|
|
for q in QUALITY: # dominance is a share (0 to 1): a gap; the others: a log ratio to the usual level
|
|
|
|
|
v, med = get[q], get[q].rolling("3D", min_periods=10).median()
|
|
|
|
|
f["q_" + q] = v - med if q == "dominance" else np.log((v.clip(lower=0) + EPS) / (med.clip(lower=0) + EPS))
|
|
|
|
|
f["is_ref"] = pd.DataFrame({s: float(kinds[s] == "global_ref") for s in sensors}, index=grid)
|
|
|
|
|
f["is_local_ref"] = pd.DataFrame({s: float(kinds[s] == "ref") for s in sensors}, index=grid)
|
|
|
|
|
names = list(f)
|
|
|
|
|
return names, np.stack([f[n].to_numpy(float) for n in names], axis=-1), f
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def station_features(f, kinds, grid, camera, temperature, measurements):
|
|
|
|
|
"""-> (names, Xc [T, Fc]) on the grid."""
|
|
|
|
|
sensors = list(kinds)
|
|
|
|
|
refs = [s for s in sensors if kinds[s] == "global_ref"]
|
|
|
|
|
targets = [s for s in sensors if kinds[s] != "global_ref"]
|
|
|
|
|
c = {}
|
|
|
|
|
ev_z = f["ev_z"]
|
|
|
|
|
c["ev_z_med"], c["ev_z_q90"], c["ev_z_max"] = ev_z.median(axis=1), ev_z.quantile(0.9, axis=1), ev_z.max(axis=1)
|
|
|
|
|
c["present_frac"] = f["present"].mean(axis=1)
|
|
|
|
|
c["rej_targets"] = f["rejected"][targets].mean(axis=1)
|
|
|
|
|
c["rej_refs"] = f["rejected"][refs].mean(axis=1) if refs else pd.Series(np.nan, grid)
|
|
|
|
|
for axis in "xy":
|
|
|
|
|
cm = f[f"d{axis}"][targets].median(axis=1)
|
|
|
|
|
c[f"common_{axis}"] = cm
|
|
|
|
|
c[f"common_j{axis}"] = f[f"j{axis}"][targets].median(axis=1)
|
|
|
|
|
c[f"spread_{axis}"] = f[f"r{axis}"][targets].abs().median(axis=1)
|
|
|
|
|
for col in ("sigma0", "reproj_err", "inlier"):
|
|
|
|
|
w = wide(measurements, col).reindex(index=grid)
|
|
|
|
|
c[col] = (w[refs] if col != "sigma0" and refs else w).mean(axis=1)
|
|
|
|
|
c["sigma0_z"] = robust_z(c["sigma0"].to_frame()).iloc[:, 0]
|
|
|
|
|
pose = pd.DataFrame(camera).set_index("t") if camera else pd.DataFrame(columns=["pan", "tilt", "rotation", "scale"])
|
|
|
|
|
if len(pose):
|
|
|
|
|
pose.index = pd.DatetimeIndex(pose.index).floor("min")
|
|
|
|
|
pose = pose[~pose.index.duplicated(keep="last")].reindex(grid)
|
|
|
|
|
else:
|
|
|
|
|
pose = pd.DataFrame(np.nan, grid, ["pan", "tilt", "rotation", "scale"])
|
|
|
|
|
pz = robust_z(pose[["pan", "tilt", "rotation", "scale"]])
|
|
|
|
|
for k in pz:
|
|
|
|
|
c["pose_" + k] = pz[k]
|
|
|
|
|
c["pose_present"] = pose["pan"].notna().astype(float)
|
|
|
|
|
if temperature:
|
|
|
|
|
temp = pd.Series(dict(temperature)).sort_index()
|
|
|
|
|
temp.index = pd.DatetimeIndex(temp.index)
|
|
|
|
|
temp = temp.reindex(temp.index.union(grid)).interpolate("time", limit=3).reindex(grid)
|
|
|
|
|
else:
|
|
|
|
|
temp = pd.Series(np.nan, grid)
|
|
|
|
|
c["temp"] = temp
|
|
|
|
|
c["temp_dev"] = temp - temp.rolling("3D", min_periods=10).median()
|
|
|
|
|
c["temp_rate"] = (temp - roll(temp.to_frame(), "1h", "mean").iloc[:, 0])
|
|
|
|
|
hours = grid.hour + grid.minute / 60
|
|
|
|
|
c["hour_sin"], c["hour_cos"] = np.sin(2 * np.pi * hours / 24), np.cos(2 * np.pi * hours / 24)
|
|
|
|
|
step = pd.Series(grid, index=grid).diff().dt.total_seconds() / 3600
|
|
|
|
|
c["log_step_h"] = np.log(step.rolling("3D", min_periods=1).median())
|
|
|
|
|
for w in ("1h", "6h"):
|
|
|
|
|
c[f"rej_targets_b{w}"] = roll(c["rej_targets"].to_frame(), w, "mean").iloc[:, 0]
|
|
|
|
|
c[f"rej_targets_f{w}"] = roll(c["rej_targets"].to_frame(), w, "mean", forward=True).iloc[:, 0]
|
|
|
|
|
names = list(c)
|
|
|
|
|
return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1)
|
|
|
|
|
|
|
|
|
|
|
2026-11-17 09:51:56 +01:00
|
|
|
def build(folder, labels, start, end, data, measurements=None):
|
|
|
|
|
"""A station folder, its labels and the range [start, end) (iso) -> dict of arrays. measurements: the station
|
|
|
|
|
already read (data.load), when several ranges are built from it."""
|
|
|
|
|
measurements = measurements or data.load(folder)
|
2026-11-16 15:48:51 +01:00
|
|
|
kinds = {s: ("global_ref" if data.is_reference(v) else
|
|
|
|
|
"ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target")
|
|
|
|
|
for s, v in measurements.items()}
|
|
|
|
|
h = data.parse_time(start) - HISTORY
|
|
|
|
|
b = data.parse_time(end)
|
|
|
|
|
measurements = {s: k for s, v in measurements.items() if (k := [m for m in v if h <= m["t"] < b])}
|
|
|
|
|
kinds = {s: kinds[s] for s in measurements}
|
|
|
|
|
grid = pd.DatetimeIndex(sorted({m["t"].replace(second=0, microsecond=0) for v in measurements.values() for m in v}))
|
|
|
|
|
s_names, xs, f = sensor_features(measurements, kinds, grid)
|
|
|
|
|
c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements)
|
|
|
|
|
keep = np.array([start <= data.iso(t) < end for t in grid])
|
|
|
|
|
instants = [data.iso(t) for t in grid[keep]]
|
2026-11-16 16:25:03 +01:00
|
|
|
return {"start": np.array(start), "end": np.array(end), "instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
|
2026-11-16 15:48:51 +01:00
|
|
|
"xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names),
|
|
|
|
|
"xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names),
|
|
|
|
|
"y": label_families(labels, instants), "families": np.array(FAMILIES)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
2026-11-17 09:51:56 +01:00
|
|
|
p.add_argument("--detectors", required=True, help="the detectors repository (scripts/, bench/; the reader is tools/scripts/data.py)")
|
2026-11-16 15:48:51 +01:00
|
|
|
p.add_argument("--data", required=True, help="folder of the station folders")
|
|
|
|
|
p.add_argument("--labels", required=True, help="folder of the <STATION>.jsonl")
|
|
|
|
|
p.add_argument("--out", required=True)
|
|
|
|
|
p.add_argument("--stations")
|
|
|
|
|
p.add_argument("--from", dest="start")
|
|
|
|
|
p.add_argument("--to", dest="end")
|
|
|
|
|
a = p.parse_args()
|
|
|
|
|
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
|
2026-11-17 09:51:56 +01:00
|
|
|
from score import read_labels # noqa: E402 (puts the tools repository's reader on the path)
|
2026-11-16 15:48:51 +01:00
|
|
|
import data # noqa: E402 the reading of the benchmark
|
|
|
|
|
from regression import label_range # noqa: E402 the labelled range, as the benchmark
|
|
|
|
|
out = Path(a.out)
|
|
|
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
for path in sorted(Path(a.labels).glob("*.jsonl")):
|
|
|
|
|
station = path.stem
|
|
|
|
|
folder = Path(a.data) / station
|
|
|
|
|
if (a.stations and station not in a.stations.split(",")) or not folder.is_dir():
|
|
|
|
|
continue
|
|
|
|
|
start, end = ((data.iso(data.parse_time(a.start)), data.iso(data.parse_time(a.end))) if a.start
|
|
|
|
|
else label_range(path))
|
|
|
|
|
d = build(str(folder), read_labels(path, start, end), start, end, data)
|
|
|
|
|
np.savez_compressed(out / f"{station}.npz", **d)
|
|
|
|
|
counts = dict(zip(FAMILIES, d["y"].sum(0).tolist()))
|
|
|
|
|
print(f"{station}: {start} → {end}, {len(d['instants'])} instants, {len(d['sensors'])} sensors, "
|
|
|
|
|
f"labelled instants {json.dumps(counts)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|