diff --git a/dataset.py b/dataset.py new file mode 100644 index 0000000..e633ff5 --- /dev/null +++ b/dataset.py @@ -0,0 +1,210 @@ +"""The dataset of the pilot: for each labelled station, the measurements of its labelled range turned into arrays. + +Per instant (a minute where at least one sensor has a measurement, as bench/score.py), per sensor: + Xs [T, S, Fs]: each sensor against its own usual level (rolling median and IQR over the HISTORY before), on the + instant and on windows in hours before and after (the step goes from 5 min to 4 h); NaN where it does not exist. +Per instant, for the station: + Xc [T, Fc]: what the sensors do together (shares, quantiles, common mode), the references, the camera pose, the + housing temperature, the time of day, the step. +Per instant, the families of the labels covering it, with the conventions of bench/score.py (bounds excluded, except a +hole without a measurement inside): + Y [T, K], over the FAMILIES. + +The measurements are read through detectors/scripts/data.py, the reading of the benchmark. One .npz per station in --out. +Usage: python dataset.py --detectors --data --labels --out + [--stations A,B] [--from YYYY-MM-DD --to YYYY-MM-DD] +""" +import argparse +import json +import sys +from datetime import timedelta +from pathlib import Path + +import numpy as np +import pandas as pd + +FAMILIES = ["Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"] +HISTORY = timedelta(days=3) # as the detectors: the usual level of a sensor is read on the 3 days before +QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"] # image quality, recent cameras only +EPS = 1e-9 + + +def label_families(labels, instants): + """Labels and sorted iso instants -> Y [T, K] uint8, as bench/score.py: a label covers the instants strictly inside + its bounds; a label with no instant strictly inside (a hole) covers its bounds too.""" + y = np.zeros((len(instants), len(FAMILIES)), np.uint8) + arr = np.array(instants) + for e in labels: + if e.get("family") not in FAMILIES: + continue + k = FAMILIES.index(e["family"]) + inside = (arr > e["start"]) & (arr < e["end"]) + y[inside if inside.any() else (arr >= e["start"]) & (arr <= e["end"]), k] = 1 + return y + + +def robust_z(df, window="3D"): + """Each column against its rolling median and IQR over the window before (current included).""" + r = df.rolling(window, min_periods=10) + med, iqr = r.median(), r.quantile(0.75) - r.quantile(0.25) + return (df - med) / (iqr / 1.349 + EPS) + + +def roll(df, window, how, forward=False): + """Time window strictly before each instant, or strictly after it (forward) -> the same frame.""" + if not forward: + return getattr(df.rolling(window, closed="left", min_periods=1), how)() + rev = df.iloc[::-1] + rev.index = df.index[-1] - rev.index # increasing distance to the end + out = getattr(rev.rolling(window, closed="left", min_periods=1), how)().iloc[::-1] + out.index = df.index + return out + + +def wide(measurements, column): + """{sensor: [measurement]} -> DataFrame [minute, sensor] of one column.""" + rows = [(m["t"].replace(second=0, microsecond=0), s, m[column]) for s, v in measurements.items() for m in v] + df = pd.DataFrame(rows, columns=["t", "s", "v"]).astype({"v": float}) + return df.pivot_table(index="t", columns="s", values="v", aggfunc="last", dropna=False) + + +def sensor_features(measurements, kinds, grid): + """-> (names, Xs [T, S, Fs]) on the grid (a DatetimeIndex, minutes, history included).""" + sensors = list(measurements) + get = {c: wide(measurements, c).reindex(index=grid, columns=sensors) for c in + ["error_value", "error_threshold", "corr_x", "corr_y"] + QUALITY} + ev, thr = get["error_value"], get["error_threshold"] + present = (ev.notna() | get["corr_x"].notna()).astype(float) + rej = (ev > thr).astype(float).where(ev.notna() & thr.notna()) + ev_z = robust_z(ev) + f = {"present": present, "rejected": rej, "ev_z": ev_z, "ev_thr": ev / thr, + "ev_logmed": np.log((ev + EPS) / (ev.rolling("3D", min_periods=10).median() + EPS))} + targets = [s for s in sensors if kinds[s] != "global_ref"] + for axis in "xy": + c = get["corr_" + axis] + jump = c.ffill().diff().where(c.notna()) # against the last valid measurement + noise = jump.abs().rolling("3D", min_periods=10).median() / 0.6745 + EPS # MAD of the steps, in sigma + dev = (c - c.rolling("3D", min_periods=10).median()) / noise + common = dev[targets].median(axis=1) + f[f"d{axis}"] = dev + f[f"j{axis}"] = jump / noise + f[f"r{axis}"] = dev.sub(common, axis=0) # what the common mode does not explain + f[f"step_{axis}_6h"] = roll(dev, "6h", "median", forward=True) - roll(dev, "6h", "median") + f[f"noise_{axis}_1h"] = roll(jump / noise, "1h", "std") + for w in ("1h", "6h"): + f[f"ev_z_b{w}"] = roll(ev_z, w, "mean") + f[f"ev_z_f{w}"] = roll(ev_z, w, "mean", forward=True) + f[f"rej_b{w}"] = roll(rej, w, "mean") + f[f"rej_f{w}"] = roll(rej, w, "mean", forward=True) + for q in QUALITY: # dominance is a share (0 to 1): a gap; the others: a log ratio to the usual level + v, med = get[q], get[q].rolling("3D", min_periods=10).median() + f["q_" + q] = v - med if q == "dominance" else np.log((v.clip(lower=0) + EPS) / (med.clip(lower=0) + EPS)) + f["is_ref"] = pd.DataFrame({s: float(kinds[s] == "global_ref") for s in sensors}, index=grid) + f["is_local_ref"] = pd.DataFrame({s: float(kinds[s] == "ref") for s in sensors}, index=grid) + names = list(f) + return names, np.stack([f[n].to_numpy(float) for n in names], axis=-1), f + + +def station_features(f, kinds, grid, camera, temperature, measurements): + """-> (names, Xc [T, Fc]) on the grid.""" + sensors = list(kinds) + refs = [s for s in sensors if kinds[s] == "global_ref"] + targets = [s for s in sensors if kinds[s] != "global_ref"] + c = {} + ev_z = f["ev_z"] + c["ev_z_med"], c["ev_z_q90"], c["ev_z_max"] = ev_z.median(axis=1), ev_z.quantile(0.9, axis=1), ev_z.max(axis=1) + c["present_frac"] = f["present"].mean(axis=1) + c["rej_targets"] = f["rejected"][targets].mean(axis=1) + c["rej_refs"] = f["rejected"][refs].mean(axis=1) if refs else pd.Series(np.nan, grid) + for axis in "xy": + cm = f[f"d{axis}"][targets].median(axis=1) + c[f"common_{axis}"] = cm + c[f"common_j{axis}"] = f[f"j{axis}"][targets].median(axis=1) + c[f"spread_{axis}"] = f[f"r{axis}"][targets].abs().median(axis=1) + for col in ("sigma0", "reproj_err", "inlier"): + w = wide(measurements, col).reindex(index=grid) + c[col] = (w[refs] if col != "sigma0" and refs else w).mean(axis=1) + c["sigma0_z"] = robust_z(c["sigma0"].to_frame()).iloc[:, 0] + pose = pd.DataFrame(camera).set_index("t") if camera else pd.DataFrame(columns=["pan", "tilt", "rotation", "scale"]) + if len(pose): + pose.index = pd.DatetimeIndex(pose.index).floor("min") + pose = pose[~pose.index.duplicated(keep="last")].reindex(grid) + else: + pose = pd.DataFrame(np.nan, grid, ["pan", "tilt", "rotation", "scale"]) + pz = robust_z(pose[["pan", "tilt", "rotation", "scale"]]) + for k in pz: + c["pose_" + k] = pz[k] + c["pose_present"] = pose["pan"].notna().astype(float) + if temperature: + temp = pd.Series(dict(temperature)).sort_index() + temp.index = pd.DatetimeIndex(temp.index) + temp = temp.reindex(temp.index.union(grid)).interpolate("time", limit=3).reindex(grid) + else: + temp = pd.Series(np.nan, grid) + c["temp"] = temp + c["temp_dev"] = temp - temp.rolling("3D", min_periods=10).median() + c["temp_rate"] = (temp - roll(temp.to_frame(), "1h", "mean").iloc[:, 0]) + hours = grid.hour + grid.minute / 60 + c["hour_sin"], c["hour_cos"] = np.sin(2 * np.pi * hours / 24), np.cos(2 * np.pi * hours / 24) + step = pd.Series(grid, index=grid).diff().dt.total_seconds() / 3600 + c["log_step_h"] = np.log(step.rolling("3D", min_periods=1).median()) + for w in ("1h", "6h"): + c[f"rej_targets_b{w}"] = roll(c["rej_targets"].to_frame(), w, "mean").iloc[:, 0] + c[f"rej_targets_f{w}"] = roll(c["rej_targets"].to_frame(), w, "mean", forward=True).iloc[:, 0] + names = list(c) + return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1) + + +def build(folder, labels, start, end, data): + """A station folder, its labels and the range [start, end) (iso) -> dict of arrays.""" + measurements = data.load(folder) + kinds = {s: ("global_ref" if data.is_reference(v) else + "ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target") + for s, v in measurements.items()} + h = data.parse_time(start) - HISTORY + b = data.parse_time(end) + measurements = {s: k for s, v in measurements.items() if (k := [m for m in v if h <= m["t"] < b])} + kinds = {s: kinds[s] for s in measurements} + grid = pd.DatetimeIndex(sorted({m["t"].replace(second=0, microsecond=0) for v in measurements.values() for m in v})) + s_names, xs, f = sensor_features(measurements, kinds, grid) + c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements) + keep = np.array([start <= data.iso(t) < end for t in grid]) + instants = [data.iso(t) for t in grid[keep]] + return {"instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())), + "xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names), + "xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names), + "y": label_families(labels, instants), "families": np.array(FAMILIES)} + + +def main(): + p = argparse.ArgumentParser(usage=__doc__) + p.add_argument("--detectors", required=True, help="the detectors repository (scripts/data.py, bench/)") + p.add_argument("--data", required=True, help="folder of the station folders") + p.add_argument("--labels", required=True, help="folder of the .jsonl") + p.add_argument("--out", required=True) + p.add_argument("--stations") + p.add_argument("--from", dest="start") + p.add_argument("--to", dest="end") + a = p.parse_args() + sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] + import data # noqa: E402 the reading of the benchmark + from regression import label_range # noqa: E402 the labelled range, as the benchmark + from score import read_labels # noqa: E402 + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + for path in sorted(Path(a.labels).glob("*.jsonl")): + station = path.stem + folder = Path(a.data) / station + if (a.stations and station not in a.stations.split(",")) or not folder.is_dir(): + continue + start, end = ((data.iso(data.parse_time(a.start)), data.iso(data.parse_time(a.end))) if a.start + else label_range(path)) + d = build(str(folder), read_labels(path, start, end), start, end, data) + np.savez_compressed(out / f"{station}.npz", **d) + counts = dict(zip(FAMILIES, d["y"].sum(0).tolist())) + print(f"{station}: {start} → {end}, {len(d['instants'])} instants, {len(d['sensors'])} sensors, " + f"labelled instants {json.dumps(counts)}") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5da331c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +numpy +pandas diff --git a/test_dataset.py b/test_dataset.py new file mode 100644 index 0000000..7ee1054 --- /dev/null +++ b/test_dataset.py @@ -0,0 +1,59 @@ +"""Checks of dataset.py on a synthetic station: python test_dataset.py .""" +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(sys.argv[1]) / "scripts")) +import data # noqa: E402 +import dataset # noqa: E402 + +# Labels as bench/score.py: bounds excluded; a hole (no instant inside) keeps its bounds. +instants = ["2026-10-01T00:00Z", "2026-10-01T00:15Z", "2026-10-01T00:30Z", "2026-10-01T01:30Z"] +y = dataset.label_families([ + {"start": "2026-10-01T00:00Z", "end": "2026-10-01T00:30Z", "family": "Visibility"}, + {"start": "2026-10-01T00:30Z", "end": "2026-10-01T01:30Z", "family": "No data"}, + {"start": "2026-10-01T00:00Z", "end": "2026-10-01T01:30Z", "family": "None"}, +], instants) +assert y[:, 0].tolist() == [0, 1, 0, 0], y +assert y[:, 5].tolist() == [0, 0, 1, 1], y + +# Four days at 15 min, two targets and a reference; T1 jumps by 10 px on the last day, all rejected at one instant. +t0 = datetime(2026, 10, 1, tzinfo=timezone.utc) +rng = np.random.default_rng(0) +measurements = {} +for s, kind in (("T1", "target"), ("T2", "target"), ("R1", "global_ref")): + v = [] + for i in range(4 * 96): + t = t0 + timedelta(minutes=15 * i, seconds=7) + x = rng.normal(0, 0.05) + (10 if s == "T1" and i >= 3 * 96 + 10 else 0) + ev = 0.2 + rng.normal(0, 0.01) + (0.5 if i == 3 * 96 + 40 else 0) + m = {c: None for c in data.NUM} + m.update(t=t, pattern_type=kind, corr_x=x, corr_y=rng.normal(0, 0.05), error_value=ev, error_threshold=0.5, + sigma0=0.01) + if data.rejected(m): + m["corr_x"] = m["corr_y"] = None + v.append(m) + measurements[s] = v + + +class Data: # data.py with the synthetic station + load = staticmethod(lambda folder: measurements) + load_camera = staticmethod(lambda folder: []) + load_temperature = staticmethod(lambda folder: []) + is_reference, parse_time, iso = staticmethod(data.is_reference), staticmethod(data.parse_time), staticmethod(data.iso) + + +d = dataset.build("-", [], "2026-10-04T00:00Z", "2026-10-05T00:00Z", Data) +names, sensors = list(d["xs_names"]), list(d["sensors"]) +assert d["xs"].shape == (96, 3, len(names)) and d["xc"].shape[0] == 96 and d["y"].sum() == 0 +assert d["instants"][0] == "2026-10-04T00:00Z" and list(d["kinds"]) == ["target", "target", "global_ref"] +jx, rx, step = (d["xs"][:, sensors.index("T1"), names.index(k)] for k in ("jx", "rx", "step_x_6h")) +assert abs(jx[10]) > 50 and np.nanmax(np.abs(jx[:10])) < 5, jx[:12] # the jump, in noise units +assert step[9] > 50 and abs(step[60]) < 10, (step[9], step[60]) # a step seen 6 h before and after +assert np.nanmin(rx[10:30]) > 50 # the other target does not move: residual +rej = d["xs"][40, :, names.index("rejected")] +assert rej.tolist() == [1, 1, 1] and d["xc"][40, list(d["xc_names"]).index("rej_targets")] == 1 +assert np.isnan(d["xs"][:, :, names.index("q_snr")]).all() # no image quality on this camera +print("ok")