"""The dataset of the pilot: for each labelled station, the measurements of its labelled range turned into arrays. Per instant (a minute where at least one sensor has a measurement, as bench/score.py), per sensor: Xs [T, S, Fs]: each sensor against its own usual level (rolling median and IQR over the HISTORY before), on the instant and on windows in hours before and after (the step goes from 5 min to 4 h); NaN where it does not exist. Per instant, for the station: Xc [T, Fc]: what the sensors do together (shares, quantiles, common mode), the references, the camera pose, the housing temperature, the time of day, the step. Per instant, the families of the labels covering it, with the conventions of bench/score.py (bounds excluded, except a hole without a measurement inside): Y [T, K], over the FAMILIES. The measurements are read through detectors/scripts/data.py, the reading of the benchmark. One .npz per station in --out. Usage: python dataset.py --detectors --data --labels --out [--stations A,B] [--from YYYY-MM-DD --to YYYY-MM-DD] """ import argparse import json import sys from datetime import timedelta from pathlib import Path import numpy as np import pandas as pd FAMILIES = ["Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"] HISTORY = timedelta(days=3) # as the detectors: the usual level of a sensor is read on the 3 days before QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"] # image quality, recent cameras only EPS = 1e-9 def label_families(labels, instants): """Labels and sorted iso instants -> Y [T, K] uint8, as bench/score.py: a label covers the instants strictly inside its bounds; a label with no instant strictly inside (a hole) covers its bounds too.""" y = np.zeros((len(instants), len(FAMILIES)), np.uint8) arr = np.array(instants) for e in labels: if e.get("family") not in FAMILIES: continue k = FAMILIES.index(e["family"]) inside = (arr > e["start"]) & (arr < e["end"]) y[inside if inside.any() else (arr >= e["start"]) & (arr <= e["end"]), k] = 1 return y def robust_z(df, window="3D"): """Each column against its rolling median and IQR over the window before (current included).""" r = df.rolling(window, min_periods=10) med, iqr = r.median(), r.quantile(0.75) - r.quantile(0.25) return (df - med) / (iqr / 1.349 + EPS) def roll(df, window, how, forward=False): """Time window strictly before each instant, or strictly after it (forward) -> the same frame.""" if not forward: return getattr(df.rolling(window, closed="left", min_periods=1), how)() rev = df.iloc[::-1] rev.index = df.index[-1] - rev.index # increasing distance to the end out = getattr(rev.rolling(window, closed="left", min_periods=1), how)().iloc[::-1] out.index = df.index return out def wide(measurements, column): """{sensor: [measurement]} -> DataFrame [minute, sensor] of one column.""" rows = [(m["t"].replace(second=0, microsecond=0), s, m[column]) for s, v in measurements.items() for m in v] df = pd.DataFrame(rows, columns=["t", "s", "v"]).astype({"v": float}) return df.pivot_table(index="t", columns="s", values="v", aggfunc="last", dropna=False) def sensor_features(measurements, kinds, grid): """-> (names, Xs [T, S, Fs]) on the grid (a DatetimeIndex, minutes, history included).""" sensors = list(measurements) get = {c: wide(measurements, c).reindex(index=grid, columns=sensors) for c in ["error_value", "error_threshold", "corr_x", "corr_y"] + QUALITY} ev, thr = get["error_value"], get["error_threshold"] present = (ev.notna() | get["corr_x"].notna()).astype(float) rej = (ev > thr).astype(float).where(ev.notna() & thr.notna()) ev_z = robust_z(ev) f = {"present": present, "rejected": rej, "ev_z": ev_z, "ev_thr": ev / thr, "ev_logmed": np.log((ev + EPS) / (ev.rolling("3D", min_periods=10).median() + EPS))} targets = [s for s in sensors if kinds[s] != "global_ref"] for axis in "xy": c = get["corr_" + axis] jump = c.ffill().diff().where(c.notna()) # against the last valid measurement noise = jump.abs().rolling("3D", min_periods=10).median() / 0.6745 + EPS # MAD of the steps, in sigma dev = (c - c.rolling("3D", min_periods=10).median()) / noise common = dev[targets].median(axis=1) f[f"d{axis}"] = dev f[f"j{axis}"] = jump / noise f[f"r{axis}"] = dev.sub(common, axis=0) # what the common mode does not explain f[f"step_{axis}_6h"] = roll(dev, "6h", "median", forward=True) - roll(dev, "6h", "median") f[f"noise_{axis}_1h"] = roll(jump / noise, "1h", "std") for w in ("1h", "6h"): f[f"ev_z_b{w}"] = roll(ev_z, w, "mean") f[f"ev_z_f{w}"] = roll(ev_z, w, "mean", forward=True) f[f"rej_b{w}"] = roll(rej, w, "mean") f[f"rej_f{w}"] = roll(rej, w, "mean", forward=True) for q in QUALITY: # dominance is a share (0 to 1): a gap; the others: a log ratio to the usual level v, med = get[q], get[q].rolling("3D", min_periods=10).median() f["q_" + q] = v - med if q == "dominance" else np.log((v.clip(lower=0) + EPS) / (med.clip(lower=0) + EPS)) f["is_ref"] = pd.DataFrame({s: float(kinds[s] == "global_ref") for s in sensors}, index=grid) f["is_local_ref"] = pd.DataFrame({s: float(kinds[s] == "ref") for s in sensors}, index=grid) names = list(f) return names, np.stack([f[n].to_numpy(float) for n in names], axis=-1), f def station_features(f, kinds, grid, camera, temperature, measurements): """-> (names, Xc [T, Fc]) on the grid.""" sensors = list(kinds) refs = [s for s in sensors if kinds[s] == "global_ref"] targets = [s for s in sensors if kinds[s] != "global_ref"] c = {} ev_z = f["ev_z"] c["ev_z_med"], c["ev_z_q90"], c["ev_z_max"] = ev_z.median(axis=1), ev_z.quantile(0.9, axis=1), ev_z.max(axis=1) c["present_frac"] = f["present"].mean(axis=1) c["rej_targets"] = f["rejected"][targets].mean(axis=1) c["rej_refs"] = f["rejected"][refs].mean(axis=1) if refs else pd.Series(np.nan, grid) for axis in "xy": cm = f[f"d{axis}"][targets].median(axis=1) c[f"common_{axis}"] = cm c[f"common_j{axis}"] = f[f"j{axis}"][targets].median(axis=1) c[f"spread_{axis}"] = f[f"r{axis}"][targets].abs().median(axis=1) for col in ("sigma0", "reproj_err", "inlier"): w = wide(measurements, col).reindex(index=grid) c[col] = (w[refs] if col != "sigma0" and refs else w).mean(axis=1) c["sigma0_z"] = robust_z(c["sigma0"].to_frame()).iloc[:, 0] pose = pd.DataFrame(camera).set_index("t") if camera else pd.DataFrame(columns=["pan", "tilt", "rotation", "scale"]) if len(pose): pose.index = pd.DatetimeIndex(pose.index).floor("min") pose = pose[~pose.index.duplicated(keep="last")].reindex(grid) else: pose = pd.DataFrame(np.nan, grid, ["pan", "tilt", "rotation", "scale"]) pz = robust_z(pose[["pan", "tilt", "rotation", "scale"]]) for k in pz: c["pose_" + k] = pz[k] c["pose_present"] = pose["pan"].notna().astype(float) if temperature: temp = pd.Series(dict(temperature)).sort_index() temp.index = pd.DatetimeIndex(temp.index) temp = temp.reindex(temp.index.union(grid)).interpolate("time", limit=3).reindex(grid) else: temp = pd.Series(np.nan, grid) c["temp"] = temp c["temp_dev"] = temp - temp.rolling("3D", min_periods=10).median() c["temp_rate"] = (temp - roll(temp.to_frame(), "1h", "mean").iloc[:, 0]) hours = grid.hour + grid.minute / 60 c["hour_sin"], c["hour_cos"] = np.sin(2 * np.pi * hours / 24), np.cos(2 * np.pi * hours / 24) step = pd.Series(grid, index=grid).diff().dt.total_seconds() / 3600 c["log_step_h"] = np.log(step.rolling("3D", min_periods=1).median()) for w in ("1h", "6h"): c[f"rej_targets_b{w}"] = roll(c["rej_targets"].to_frame(), w, "mean").iloc[:, 0] c[f"rej_targets_f{w}"] = roll(c["rej_targets"].to_frame(), w, "mean", forward=True).iloc[:, 0] names = list(c) return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1) def build(folder, labels, start, end, data): """A station folder, its labels and the range [start, end) (iso) -> dict of arrays.""" measurements = data.load(folder) kinds = {s: ("global_ref" if data.is_reference(v) else "ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target") for s, v in measurements.items()} h = data.parse_time(start) - HISTORY b = data.parse_time(end) measurements = {s: k for s, v in measurements.items() if (k := [m for m in v if h <= m["t"] < b])} kinds = {s: kinds[s] for s in measurements} grid = pd.DatetimeIndex(sorted({m["t"].replace(second=0, microsecond=0) for v in measurements.values() for m in v})) s_names, xs, f = sensor_features(measurements, kinds, grid) c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements) keep = np.array([start <= data.iso(t) < end for t in grid]) instants = [data.iso(t) for t in grid[keep]] return {"instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())), "xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names), "xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names), "y": label_families(labels, instants), "families": np.array(FAMILIES)} def main(): p = argparse.ArgumentParser(usage=__doc__) p.add_argument("--detectors", required=True, help="the detectors repository (scripts/data.py, bench/)") p.add_argument("--data", required=True, help="folder of the station folders") p.add_argument("--labels", required=True, help="folder of the .jsonl") p.add_argument("--out", required=True) p.add_argument("--stations") p.add_argument("--from", dest="start") p.add_argument("--to", dest="end") a = p.parse_args() sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] import data # noqa: E402 the reading of the benchmark from regression import label_range # noqa: E402 the labelled range, as the benchmark from score import read_labels # noqa: E402 out = Path(a.out) out.mkdir(parents=True, exist_ok=True) for path in sorted(Path(a.labels).glob("*.jsonl")): station = path.stem folder = Path(a.data) / station if (a.stations and station not in a.stations.split(",")) or not folder.is_dir(): continue start, end = ((data.iso(data.parse_time(a.start)), data.iso(data.parse_time(a.end))) if a.start else label_range(path)) d = build(str(folder), read_labels(path, start, end), start, end, data) np.savez_compressed(out / f"{station}.npz", **d) counts = dict(zip(FAMILIES, d["y"].sum(0).tolist())) print(f"{station}: {start} → {end}, {len(d['instants'])} instants, {len(d['sensors'])} sensors, " f"labelled instants {json.dumps(counts)}") if __name__ == "__main__": main()