Files

211 lines
11 KiB
Python
Raw Permalink Normal View History

"""The dataset of the pilot: for each labelled station, the measurements of its labelled range turned into arrays.
Per instant (a minute where at least one sensor has a measurement, as bench/score.py), per sensor:
Xs [T, S, Fs]: each sensor against its own usual level (rolling median and IQR over the HISTORY before), on the
instant and on windows in hours before and after (the step goes from 5 min to 4 h); NaN where it does not exist.
Per instant, for the station:
Xc [T, Fc]: what the sensors do together (shares, quantiles, common mode), the references, the camera pose, the
housing temperature, the time of day, the step.
Per instant, the families of the labels covering it, with the conventions of bench/score.py (bounds excluded, except a
hole without a measurement inside):
Y [T, K], over the FAMILIES.
The measurements are read through detectors/scripts/data.py, the reading of the benchmark. One .npz per station in --out.
Usage: python dataset.py --detectors <detectors> --data <data> --labels <labels folder> --out <folder>
[--stations A,B] [--from YYYY-MM-DD --to YYYY-MM-DD]
"""
import argparse
import json
import sys
from datetime import timedelta
from pathlib import Path
import numpy as np
import pandas as pd
FAMILIES = ["Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data"]
HISTORY = timedelta(days=3) # as the detectors: the usual level of a sensor is read on the 3 days before
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"] # image quality, recent cameras only
EPS = 1e-9
def label_families(labels, instants):
"""Labels and sorted iso instants -> Y [T, K] uint8, as bench/score.py: a label covers the instants strictly inside
its bounds; a label with no instant strictly inside (a hole) covers its bounds too."""
y = np.zeros((len(instants), len(FAMILIES)), np.uint8)
arr = np.array(instants)
for e in labels:
if e.get("family") not in FAMILIES:
continue
k = FAMILIES.index(e["family"])
inside = (arr > e["start"]) & (arr < e["end"])
y[inside if inside.any() else (arr >= e["start"]) & (arr <= e["end"]), k] = 1
return y
def robust_z(df, window="3D"):
"""Each column against its rolling median and IQR over the window before (current included)."""
r = df.rolling(window, min_periods=10)
med, iqr = r.median(), r.quantile(0.75) - r.quantile(0.25)
return (df - med) / (iqr / 1.349 + EPS)
def roll(df, window, how, forward=False):
"""Time window strictly before each instant, or strictly after it (forward) -> the same frame."""
if not forward:
return getattr(df.rolling(window, closed="left", min_periods=1), how)()
rev = df.iloc[::-1]
rev.index = df.index[-1] - rev.index # increasing distance to the end
out = getattr(rev.rolling(window, closed="left", min_periods=1), how)().iloc[::-1]
out.index = df.index
return out
def wide(measurements, column):
"""{sensor: [measurement]} -> DataFrame [minute, sensor] of one column."""
rows = [(m["t"].replace(second=0, microsecond=0), s, m[column]) for s, v in measurements.items() for m in v]
df = pd.DataFrame(rows, columns=["t", "s", "v"]).astype({"v": float})
return df.pivot_table(index="t", columns="s", values="v", aggfunc="last", dropna=False)
def sensor_features(measurements, kinds, grid):
"""-> (names, Xs [T, S, Fs]) on the grid (a DatetimeIndex, minutes, history included)."""
sensors = list(measurements)
get = {c: wide(measurements, c).reindex(index=grid, columns=sensors) for c in
["error_value", "error_threshold", "corr_x", "corr_y"] + QUALITY}
ev, thr = get["error_value"], get["error_threshold"]
present = (ev.notna() | get["corr_x"].notna()).astype(float)
rej = (ev > thr).astype(float).where(ev.notna() & thr.notna())
ev_z = robust_z(ev)
f = {"present": present, "rejected": rej, "ev_z": ev_z, "ev_thr": ev / thr,
"ev_logmed": np.log((ev + EPS) / (ev.rolling("3D", min_periods=10).median() + EPS))}
targets = [s for s in sensors if kinds[s] != "global_ref"]
for axis in "xy":
c = get["corr_" + axis]
jump = c.ffill().diff().where(c.notna()) # against the last valid measurement
noise = jump.abs().rolling("3D", min_periods=10).median() / 0.6745 + EPS # MAD of the steps, in sigma
dev = (c - c.rolling("3D", min_periods=10).median()) / noise
common = dev[targets].median(axis=1)
f[f"d{axis}"] = dev
f[f"j{axis}"] = jump / noise
f[f"r{axis}"] = dev.sub(common, axis=0) # what the common mode does not explain
f[f"step_{axis}_6h"] = roll(dev, "6h", "median", forward=True) - roll(dev, "6h", "median")
f[f"noise_{axis}_1h"] = roll(jump / noise, "1h", "std")
for w in ("1h", "6h"):
f[f"ev_z_b{w}"] = roll(ev_z, w, "mean")
f[f"ev_z_f{w}"] = roll(ev_z, w, "mean", forward=True)
f[f"rej_b{w}"] = roll(rej, w, "mean")
f[f"rej_f{w}"] = roll(rej, w, "mean", forward=True)
for q in QUALITY: # dominance is a share (0 to 1): a gap; the others: a log ratio to the usual level
v, med = get[q], get[q].rolling("3D", min_periods=10).median()
f["q_" + q] = v - med if q == "dominance" else np.log((v.clip(lower=0) + EPS) / (med.clip(lower=0) + EPS))
f["is_ref"] = pd.DataFrame({s: float(kinds[s] == "global_ref") for s in sensors}, index=grid)
f["is_local_ref"] = pd.DataFrame({s: float(kinds[s] == "ref") for s in sensors}, index=grid)
names = list(f)
return names, np.stack([f[n].to_numpy(float) for n in names], axis=-1), f
def station_features(f, kinds, grid, camera, temperature, measurements):
"""-> (names, Xc [T, Fc]) on the grid."""
sensors = list(kinds)
refs = [s for s in sensors if kinds[s] == "global_ref"]
targets = [s for s in sensors if kinds[s] != "global_ref"]
c = {}
ev_z = f["ev_z"]
c["ev_z_med"], c["ev_z_q90"], c["ev_z_max"] = ev_z.median(axis=1), ev_z.quantile(0.9, axis=1), ev_z.max(axis=1)
c["present_frac"] = f["present"].mean(axis=1)
c["rej_targets"] = f["rejected"][targets].mean(axis=1)
c["rej_refs"] = f["rejected"][refs].mean(axis=1) if refs else pd.Series(np.nan, grid)
for axis in "xy":
cm = f[f"d{axis}"][targets].median(axis=1)
c[f"common_{axis}"] = cm
c[f"common_j{axis}"] = f[f"j{axis}"][targets].median(axis=1)
c[f"spread_{axis}"] = f[f"r{axis}"][targets].abs().median(axis=1)
for col in ("sigma0", "reproj_err", "inlier"):
w = wide(measurements, col).reindex(index=grid)
c[col] = (w[refs] if col != "sigma0" and refs else w).mean(axis=1)
c["sigma0_z"] = robust_z(c["sigma0"].to_frame()).iloc[:, 0]
pose = pd.DataFrame(camera).set_index("t") if camera else pd.DataFrame(columns=["pan", "tilt", "rotation", "scale"])
if len(pose):
pose.index = pd.DatetimeIndex(pose.index).floor("min")
pose = pose[~pose.index.duplicated(keep="last")].reindex(grid)
else:
pose = pd.DataFrame(np.nan, grid, ["pan", "tilt", "rotation", "scale"])
pz = robust_z(pose[["pan", "tilt", "rotation", "scale"]])
for k in pz:
c["pose_" + k] = pz[k]
c["pose_present"] = pose["pan"].notna().astype(float)
if temperature:
temp = pd.Series(dict(temperature)).sort_index()
temp.index = pd.DatetimeIndex(temp.index)
temp = temp.reindex(temp.index.union(grid)).interpolate("time", limit=3).reindex(grid)
else:
temp = pd.Series(np.nan, grid)
c["temp"] = temp
c["temp_dev"] = temp - temp.rolling("3D", min_periods=10).median()
c["temp_rate"] = (temp - roll(temp.to_frame(), "1h", "mean").iloc[:, 0])
hours = grid.hour + grid.minute / 60
c["hour_sin"], c["hour_cos"] = np.sin(2 * np.pi * hours / 24), np.cos(2 * np.pi * hours / 24)
step = pd.Series(grid, index=grid).diff().dt.total_seconds() / 3600
c["log_step_h"] = np.log(step.rolling("3D", min_periods=1).median())
for w in ("1h", "6h"):
c[f"rej_targets_b{w}"] = roll(c["rej_targets"].to_frame(), w, "mean").iloc[:, 0]
c[f"rej_targets_f{w}"] = roll(c["rej_targets"].to_frame(), w, "mean", forward=True).iloc[:, 0]
names = list(c)
return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1)
def build(folder, labels, start, end, data, measurements=None):
"""A station folder, its labels and the range [start, end) (iso) -> dict of arrays. measurements: the station
already read (data.load), when several ranges are built from it."""
measurements = measurements or data.load(folder)
kinds = {s: ("global_ref" if data.is_reference(v) else
"ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target")
for s, v in measurements.items()}
h = data.parse_time(start) - HISTORY
b = data.parse_time(end)
measurements = {s: k for s, v in measurements.items() if (k := [m for m in v if h <= m["t"] < b])}
kinds = {s: kinds[s] for s in measurements}
grid = pd.DatetimeIndex(sorted({m["t"].replace(second=0, microsecond=0) for v in measurements.values() for m in v}))
s_names, xs, f = sensor_features(measurements, kinds, grid)
c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements)
keep = np.array([start <= data.iso(t) < end for t in grid])
instants = [data.iso(t) for t in grid[keep]]
return {"start": np.array(start), "end": np.array(end), "instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
"xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names),
"xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names),
"y": label_families(labels, instants), "families": np.array(FAMILIES)}
def main():
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("--detectors", required=True, help="the detectors repository (scripts/, bench/; the reader is tools/scripts/data.py)")
p.add_argument("--data", required=True, help="folder of the station folders")
p.add_argument("--labels", required=True, help="folder of the <STATION>.jsonl")
p.add_argument("--out", required=True)
p.add_argument("--stations")
p.add_argument("--from", dest="start")
p.add_argument("--to", dest="end")
a = p.parse_args()
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
from score import read_labels # noqa: E402 (puts the tools repository's reader on the path)
import data # noqa: E402 the reading of the benchmark
from regression import label_range # noqa: E402 the labelled range, as the benchmark
out = Path(a.out)
out.mkdir(parents=True, exist_ok=True)
for path in sorted(Path(a.labels).glob("*.jsonl")):
station = path.stem
folder = Path(a.data) / station
if (a.stations and station not in a.stations.split(",")) or not folder.is_dir():
continue
start, end = ((data.iso(data.parse_time(a.start)), data.iso(data.parse_time(a.end))) if a.start
else label_range(path))
d = build(str(folder), read_labels(path, start, end), start, end, data)
np.savez_compressed(out / f"{station}.npz", **d)
counts = dict(zip(FAMILIES, d["y"].sum(0).tolist()))
print(f"{station}: {start} → {end}, {len(d['instants'])} instants, {len(d['sensors'])} sensors, "
f"labelled instants {json.dumps(counts)}")
if __name__ == "__main__":
main()