Step 1: the dataset (features per sensor and per station, families per instant as the benchmark)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9cf20766a5
commit
b6eecf3ab7
3 files changed
+271
No files matched your search
+210
@@ -0,0 +1,210 @@
|
||||
"""The dataset of the pilot: for each labelled station, the measurements of its labelled range turned into arrays.
|
||||
|
||||
Per instant (a minute where at least one sensor has a measurement, as bench/score.py), per sensor:
|
||||
Xs [T, S, Fs]: each sensor against its own usual level (rolling median and IQR over the HISTORY before), on the
|
||||
instant and on windows in hours before and after (the step goes from 5 min to 4 h); NaN where it does not exist.
|
||||
Per instant, for the station:
|
||||
Xc [T, Fc]: what the sensors do together (shares, quantiles, common mode), the references, the camera pose, the
|
||||
housing temperature, the time of day, the step.
|
||||
Per instant, the families of the labels covering it, with the conventions of bench/score.py (bounds excluded, except a
|
||||
hole without a measurement inside):
|
||||
Y [T, K], over the FAMILIES.
|
||||
|
||||
The measurements are read through detectors/scripts/data.py, the reading of the benchmark. One .npz per station in --out.
|
||||
Usage: python dataset.py --detectors <detectors> --data <data> --labels <labels folder> --out <folder>
|
||||
[--stations A,B] [--from YYYY-MM-DD --to YYYY-MM-DD]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
FAMILIES = ["Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"]
|
||||
HISTORY = timedelta(days=3) # as the detectors: the usual level of a sensor is read on the 3 days before
|
||||
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"] # image quality, recent cameras only
|
||||
EPS = 1e-9
|
||||
|
||||
|
||||
def label_families(labels, instants):
|
||||
"""Labels and sorted iso instants -> Y [T, K] uint8, as bench/score.py: a label covers the instants strictly inside
|
||||
its bounds; a label with no instant strictly inside (a hole) covers its bounds too."""
|
||||
y = np.zeros((len(instants), len(FAMILIES)), np.uint8)
|
||||
arr = np.array(instants)
|
||||
for e in labels:
|
||||
if e.get("family") not in FAMILIES:
|
||||
continue
|
||||
k = FAMILIES.index(e["family"])
|
||||
inside = (arr > e["start"]) & (arr < e["end"])
|
||||
y[inside if inside.any() else (arr >= e["start"]) & (arr <= e["end"]), k] = 1
|
||||
return y
|
||||
|
||||
|
||||
def robust_z(df, window="3D"):
|
||||
"""Each column against its rolling median and IQR over the window before (current included)."""
|
||||
r = df.rolling(window, min_periods=10)
|
||||
med, iqr = r.median(), r.quantile(0.75) - r.quantile(0.25)
|
||||
return (df - med) / (iqr / 1.349 + EPS)
|
||||
|
||||
|
||||
def roll(df, window, how, forward=False):
|
||||
"""Time window strictly before each instant, or strictly after it (forward) -> the same frame."""
|
||||
if not forward:
|
||||
return getattr(df.rolling(window, closed="left", min_periods=1), how)()
|
||||
rev = df.iloc[::-1]
|
||||
rev.index = df.index[-1] - rev.index # increasing distance to the end
|
||||
out = getattr(rev.rolling(window, closed="left", min_periods=1), how)().iloc[::-1]
|
||||
out.index = df.index
|
||||
return out
|
||||
|
||||
|
||||
def wide(measurements, column):
|
||||
"""{sensor: [measurement]} -> DataFrame [minute, sensor] of one column."""
|
||||
rows = [(m["t"].replace(second=0, microsecond=0), s, m[column]) for s, v in measurements.items() for m in v]
|
||||
df = pd.DataFrame(rows, columns=["t", "s", "v"]).astype({"v": float})
|
||||
return df.pivot_table(index="t", columns="s", values="v", aggfunc="last", dropna=False)
|
||||
|
||||
|
||||
def sensor_features(measurements, kinds, grid):
|
||||
"""-> (names, Xs [T, S, Fs]) on the grid (a DatetimeIndex, minutes, history included)."""
|
||||
sensors = list(measurements)
|
||||
get = {c: wide(measurements, c).reindex(index=grid, columns=sensors) for c in
|
||||
["error_value", "error_threshold", "corr_x", "corr_y"] + QUALITY}
|
||||
ev, thr = get["error_value"], get["error_threshold"]
|
||||
present = (ev.notna() | get["corr_x"].notna()).astype(float)
|
||||
rej = (ev > thr).astype(float).where(ev.notna() & thr.notna())
|
||||
ev_z = robust_z(ev)
|
||||
f = {"present": present, "rejected": rej, "ev_z": ev_z, "ev_thr": ev / thr,
|
||||
"ev_logmed": np.log((ev + EPS) / (ev.rolling("3D", min_periods=10).median() + EPS))}
|
||||
targets = [s for s in sensors if kinds[s] != "global_ref"]
|
||||
for axis in "xy":
|
||||
c = get["corr_" + axis]
|
||||
jump = c.ffill().diff().where(c.notna()) # against the last valid measurement
|
||||
noise = jump.abs().rolling("3D", min_periods=10).median() / 0.6745 + EPS # MAD of the steps, in sigma
|
||||
dev = (c - c.rolling("3D", min_periods=10).median()) / noise
|
||||
common = dev[targets].median(axis=1)
|
||||
f[f"d{axis}"] = dev
|
||||
f[f"j{axis}"] = jump / noise
|
||||
f[f"r{axis}"] = dev.sub(common, axis=0) # what the common mode does not explain
|
||||
f[f"step_{axis}_6h"] = roll(dev, "6h", "median", forward=True) - roll(dev, "6h", "median")
|
||||
f[f"noise_{axis}_1h"] = roll(jump / noise, "1h", "std")
|
||||
for w in ("1h", "6h"):
|
||||
f[f"ev_z_b{w}"] = roll(ev_z, w, "mean")
|
||||
f[f"ev_z_f{w}"] = roll(ev_z, w, "mean", forward=True)
|
||||
f[f"rej_b{w}"] = roll(rej, w, "mean")
|
||||
f[f"rej_f{w}"] = roll(rej, w, "mean", forward=True)
|
||||
for q in QUALITY: # dominance is a share (0 to 1): a gap; the others: a log ratio to the usual level
|
||||
v, med = get[q], get[q].rolling("3D", min_periods=10).median()
|
||||
f["q_" + q] = v - med if q == "dominance" else np.log((v.clip(lower=0) + EPS) / (med.clip(lower=0) + EPS))
|
||||
f["is_ref"] = pd.DataFrame({s: float(kinds[s] == "global_ref") for s in sensors}, index=grid)
|
||||
f["is_local_ref"] = pd.DataFrame({s: float(kinds[s] == "ref") for s in sensors}, index=grid)
|
||||
names = list(f)
|
||||
return names, np.stack([f[n].to_numpy(float) for n in names], axis=-1), f
|
||||
|
||||
|
||||
def station_features(f, kinds, grid, camera, temperature, measurements):
|
||||
"""-> (names, Xc [T, Fc]) on the grid."""
|
||||
sensors = list(kinds)
|
||||
refs = [s for s in sensors if kinds[s] == "global_ref"]
|
||||
targets = [s for s in sensors if kinds[s] != "global_ref"]
|
||||
c = {}
|
||||
ev_z = f["ev_z"]
|
||||
c["ev_z_med"], c["ev_z_q90"], c["ev_z_max"] = ev_z.median(axis=1), ev_z.quantile(0.9, axis=1), ev_z.max(axis=1)
|
||||
c["present_frac"] = f["present"].mean(axis=1)
|
||||
c["rej_targets"] = f["rejected"][targets].mean(axis=1)
|
||||
c["rej_refs"] = f["rejected"][refs].mean(axis=1) if refs else pd.Series(np.nan, grid)
|
||||
for axis in "xy":
|
||||
cm = f[f"d{axis}"][targets].median(axis=1)
|
||||
c[f"common_{axis}"] = cm
|
||||
c[f"common_j{axis}"] = f[f"j{axis}"][targets].median(axis=1)
|
||||
c[f"spread_{axis}"] = f[f"r{axis}"][targets].abs().median(axis=1)
|
||||
for col in ("sigma0", "reproj_err", "inlier"):
|
||||
w = wide(measurements, col).reindex(index=grid)
|
||||
c[col] = (w[refs] if col != "sigma0" and refs else w).mean(axis=1)
|
||||
c["sigma0_z"] = robust_z(c["sigma0"].to_frame()).iloc[:, 0]
|
||||
pose = pd.DataFrame(camera).set_index("t") if camera else pd.DataFrame(columns=["pan", "tilt", "rotation", "scale"])
|
||||
if len(pose):
|
||||
pose.index = pd.DatetimeIndex(pose.index).floor("min")
|
||||
pose = pose[~pose.index.duplicated(keep="last")].reindex(grid)
|
||||
else:
|
||||
pose = pd.DataFrame(np.nan, grid, ["pan", "tilt", "rotation", "scale"])
|
||||
pz = robust_z(pose[["pan", "tilt", "rotation", "scale"]])
|
||||
for k in pz:
|
||||
c["pose_" + k] = pz[k]
|
||||
c["pose_present"] = pose["pan"].notna().astype(float)
|
||||
if temperature:
|
||||
temp = pd.Series(dict(temperature)).sort_index()
|
||||
temp.index = pd.DatetimeIndex(temp.index)
|
||||
temp = temp.reindex(temp.index.union(grid)).interpolate("time", limit=3).reindex(grid)
|
||||
else:
|
||||
temp = pd.Series(np.nan, grid)
|
||||
c["temp"] = temp
|
||||
c["temp_dev"] = temp - temp.rolling("3D", min_periods=10).median()
|
||||
c["temp_rate"] = (temp - roll(temp.to_frame(), "1h", "mean").iloc[:, 0])
|
||||
hours = grid.hour + grid.minute / 60
|
||||
c["hour_sin"], c["hour_cos"] = np.sin(2 * np.pi * hours / 24), np.cos(2 * np.pi * hours / 24)
|
||||
step = pd.Series(grid, index=grid).diff().dt.total_seconds() / 3600
|
||||
c["log_step_h"] = np.log(step.rolling("3D", min_periods=1).median())
|
||||
for w in ("1h", "6h"):
|
||||
c[f"rej_targets_b{w}"] = roll(c["rej_targets"].to_frame(), w, "mean").iloc[:, 0]
|
||||
c[f"rej_targets_f{w}"] = roll(c["rej_targets"].to_frame(), w, "mean", forward=True).iloc[:, 0]
|
||||
names = list(c)
|
||||
return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1)
|
||||
|
||||
|
||||
def build(folder, labels, start, end, data):
|
||||
"""A station folder, its labels and the range [start, end) (iso) -> dict of arrays."""
|
||||
measurements = data.load(folder)
|
||||
kinds = {s: ("global_ref" if data.is_reference(v) else
|
||||
"ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target")
|
||||
for s, v in measurements.items()}
|
||||
h = data.parse_time(start) - HISTORY
|
||||
b = data.parse_time(end)
|
||||
measurements = {s: k for s, v in measurements.items() if (k := [m for m in v if h <= m["t"] < b])}
|
||||
kinds = {s: kinds[s] for s in measurements}
|
||||
grid = pd.DatetimeIndex(sorted({m["t"].replace(second=0, microsecond=0) for v in measurements.values() for m in v}))
|
||||
s_names, xs, f = sensor_features(measurements, kinds, grid)
|
||||
c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements)
|
||||
keep = np.array([start <= data.iso(t) < end for t in grid])
|
||||
instants = [data.iso(t) for t in grid[keep]]
|
||||
return {"instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
|
||||
"xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names),
|
||||
"xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names),
|
||||
"y": label_families(labels, instants), "families": np.array(FAMILIES)}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("--detectors", required=True, help="the detectors repository (scripts/data.py, bench/)")
|
||||
p.add_argument("--data", required=True, help="folder of the station folders")
|
||||
p.add_argument("--labels", required=True, help="folder of the <STATION>.jsonl")
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--stations")
|
||||
p.add_argument("--from", dest="start")
|
||||
p.add_argument("--to", dest="end")
|
||||
a = p.parse_args()
|
||||
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
|
||||
import data # noqa: E402 the reading of the benchmark
|
||||
from regression import label_range # noqa: E402 the labelled range, as the benchmark
|
||||
from score import read_labels # noqa: E402
|
||||
out = Path(a.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for path in sorted(Path(a.labels).glob("*.jsonl")):
|
||||
station = path.stem
|
||||
folder = Path(a.data) / station
|
||||
if (a.stations and station not in a.stations.split(",")) or not folder.is_dir():
|
||||
continue
|
||||
start, end = ((data.iso(data.parse_time(a.start)), data.iso(data.parse_time(a.end))) if a.start
|
||||
else label_range(path))
|
||||
d = build(str(folder), read_labels(path, start, end), start, end, data)
|
||||
np.savez_compressed(out / f"{station}.npz", **d)
|
||||
counts = dict(zip(FAMILIES, d["y"].sum(0).tolist()))
|
||||
print(f"{station}: {start} → {end}, {len(d['instants'])} instants, {len(d['sensors'])} sensors, "
|
||||
f"labelled instants {json.dumps(counts)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,2 @@
|
||||
numpy
|
||||
pandas
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Checks of dataset.py on a synthetic station: python test_dataset.py <detectors>."""
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(sys.argv[1]) / "scripts"))
|
||||
import data # noqa: E402
|
||||
import dataset # noqa: E402
|
||||
|
||||
# Labels as bench/score.py: bounds excluded; a hole (no instant inside) keeps its bounds.
|
||||
instants = ["2026-10-01T00:00Z", "2026-10-01T00:15Z", "2026-10-01T00:30Z", "2026-10-01T01:30Z"]
|
||||
y = dataset.label_families([
|
||||
{"start": "2026-10-01T00:00Z", "end": "2026-10-01T00:30Z", "family": "Visibility"},
|
||||
{"start": "2026-10-01T00:30Z", "end": "2026-10-01T01:30Z", "family": "No data"},
|
||||
{"start": "2026-10-01T00:00Z", "end": "2026-10-01T01:30Z", "family": "None"},
|
||||
], instants)
|
||||
assert y[:, 0].tolist() == [0, 1, 0, 0], y
|
||||
assert y[:, 5].tolist() == [0, 0, 1, 1], y
|
||||
|
||||
# Four days at 15 min, two targets and a reference; T1 jumps by 10 px on the last day, all rejected at one instant.
|
||||
t0 = datetime(2026, 10, 1, tzinfo=timezone.utc)
|
||||
rng = np.random.default_rng(0)
|
||||
measurements = {}
|
||||
for s, kind in (("T1", "target"), ("T2", "target"), ("R1", "global_ref")):
|
||||
v = []
|
||||
for i in range(4 * 96):
|
||||
t = t0 + timedelta(minutes=15 * i, seconds=7)
|
||||
x = rng.normal(0, 0.05) + (10 if s == "T1" and i >= 3 * 96 + 10 else 0)
|
||||
ev = 0.2 + rng.normal(0, 0.01) + (0.5 if i == 3 * 96 + 40 else 0)
|
||||
m = {c: None for c in data.NUM}
|
||||
m.update(t=t, pattern_type=kind, corr_x=x, corr_y=rng.normal(0, 0.05), error_value=ev, error_threshold=0.5,
|
||||
sigma0=0.01)
|
||||
if data.rejected(m):
|
||||
m["corr_x"] = m["corr_y"] = None
|
||||
v.append(m)
|
||||
measurements[s] = v
|
||||
|
||||
|
||||
class Data: # data.py with the synthetic station
|
||||
load = staticmethod(lambda folder: measurements)
|
||||
load_camera = staticmethod(lambda folder: [])
|
||||
load_temperature = staticmethod(lambda folder: [])
|
||||
is_reference, parse_time, iso = staticmethod(data.is_reference), staticmethod(data.parse_time), staticmethod(data.iso)
|
||||
|
||||
|
||||
d = dataset.build("-", [], "2026-10-04T00:00Z", "2026-10-05T00:00Z", Data)
|
||||
names, sensors = list(d["xs_names"]), list(d["sensors"])
|
||||
assert d["xs"].shape == (96, 3, len(names)) and d["xc"].shape[0] == 96 and d["y"].sum() == 0
|
||||
assert d["instants"][0] == "2026-10-04T00:00Z" and list(d["kinds"]) == ["target", "target", "global_ref"]
|
||||
jx, rx, step = (d["xs"][:, sensors.index("T1"), names.index(k)] for k in ("jx", "rx", "step_x_6h"))
|
||||
assert abs(jx[10]) > 50 and np.nanmax(np.abs(jx[:10])) < 5, jx[:12] # the jump, in noise units
|
||||
assert step[9] > 50 and abs(step[60]) < 10, (step[9], step[60]) # a step seen 6 h before and after
|
||||
assert np.nanmin(rx[10:30]) > 50 # the other target does not move: residual
|
||||
rej = d["xs"][40, :, names.index("rejected")]
|
||||
assert rej.tolist() == [1, 1, 1] and d["xc"][40, list(d["xc_names"]).index("rej_targets")] == 1
|
||||
assert np.isnan(d["xs"][:, :, names.index("q_snr")]).all() # no image quality on this camera
|
||||
print("ok")
|
||||
Reference in new issue
Block a user