"""Inventory of a station folder: what is available, what is missing. Usage: python3 tools/scripts/inventory.py data/PS_DES_COUDRES/ [--start T] [--end T] [--files] The counts cover the whole period. With --start and/or --end, one more section lists, over this window, the absent steps (dated, with the sensors that have no measurement) and, per sensor, the rejected measurements (dated, in runs of consecutive steps; rejected: error_value above the threshold, references included), then the number of references retained by the compensation (inlier = 1; inlier empty over the window: reproj_err filled) at each instant and, dated, the runs of instants where references are not (which ones, and why: rejected, inlier value, empty). Camera movements (camera/): number of poses, period, and the measurement instants without a pose (no registration), over the whole period then over the window. Before the counts, what each source contains (over the window, otherwise the whole period): the columns of the export, of the pose and of quality/, filled or empty (with, for an export column that not all sensors carry, how many carry it), the names of the system metrics of metrics/, the images and the differential images. Duplicate readings (same sensor, same instant in two exports, or twice in one) are counted per export. Images: per run of equal gaps between two images (within 5 min; over the window if it is given). A dated run that ends on the same day writes only the time of its end. Gives values, not a verdict. """ import argparse import collections import csv import glob import os import re import statistics from datetime import datetime, timedelta, timezone from data import filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected from summary import QUALITY METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_\d{2}_\d{2}") # NIEDERHORN_BusCurrent_2026_09_22_00_00 to ….csv def span(a, b, n): """A dated run: the start in full, the end without the date when it falls on the same day.""" if n == 1: return iso(a) return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})" def duplicates(paths): """Readings (same sensor, same instant) already read in a previous export or higher up in the same one: load keeps only one, that of the last file in name order. One line per export that has some.""" seen, lines = {}, [] for p in paths: n = already = different = 0 with open(p, newline="") as fh: for r in csv.DictReader(fh): n += 1 key = (r["sensor_name"].strip(), r["time_utc"]) val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None)) if key in seen: already += 1 different += seen[key] != val seen[key] = val if already: lines.append(f" {os.path.basename(p)}: {already} of {n} already read, {different} of them with different values") if lines: print("Duplicate readings (same sensor, same instant in an export already read, in name order; " "the tools keep the one of the last file):") print("\n".join(lines)) def list_paths(paths, full): """A pumping folder holds hundreds of files: beyond 5, the first and the last only.""" if full or len(paths) <= 5: shown = paths else: shown = [paths[0], paths[-1]] print(f" {len(paths)} files, first and last (--files for the full list):") for p in shown: print(f" {p}") def files(f, full=False): print(f'"advanced" exports (source of the measurements): {len(f["advanced"])}') list_paths(f["advanced"], full) duplicates(f["advanced"]) print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, " "local time without a zone; see labeling/knowledge/displacement-columns.md)") list_paths(f["hourly"], full) print(f"Camera movements (pose at each registration): {len(f['camera'])}") list_paths(f["camera"], full) print(f"Other files: {len(f['others'])}") list_paths(f["others"], full) def filled(paths, start, end): """CSVs of a source -> (rows of the window, {column: rows with a value}, {column: sensors that have one}), columns in header order; sensors: sensor_name, if there is one.""" rows, count, who = 0, {}, collections.defaultdict(set) for p in paths: with open(p, newline="") as fh: r = csv.DictReader(fh) for c in r.fieldnames or []: if c not in ("", "time_utc", "sensor_name"): count.setdefault(c, 0) for row in r: if (start or end) and filter_window([{"t": parse_time(row["time_utc"])}], start, end) == []: continue rows += 1 for c in count: if (row.get(c) or "").strip(): count[c] += 1 if row.get("sensor_name"): who[c].add(row["sensor_name"].strip()) return rows, count, who def contents(folder, f, start=None, end=None): """What each source contains over the window: its columns, filled or empty, and the names of the metrics. Names and counts, never values (metrics/ carries identifiers: SIM_CCID, ip_*).""" what = "over the window" if start or end else "over the whole period" print(f'\nWhat the data contain, {what} (per column, the rows that have a value; "empty": none):') others = collections.defaultdict(list) for p in f["others"]: others[os.path.relpath(p, folder).split(os.sep)[0]].append(p) for title, paths in (('"advanced" export', f["advanced"]), ("Camera pose (camera/)", f["camera"]), ("Image quality (quality/)", others.pop("quality", []))): if paths: n, count, who = filled(paths, start, end) everyone = set().union(*who.values()) def carriers(c): # number of sensors that carry the column, when it is not all of them return f" ({len(who[c])} sensors of {len(everyone)})" if everyone and len(who[c]) < len(everyone) else "" print(f" {title}: {n} rows; " + ", ".join(f"{c} {k}{carriers(c)}" if k else f"{c} empty" for c, k in count.items())) if paths is f["advanced"]: q = [c for c in QUALITY if count.get(c)] print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, " "labeling/knowledge/quality-metrics.md" if q else "empty")) days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window for p in others.pop("metrics", []): m = METRIC_RE.match(os.path.basename(p)) if not m: continue day = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc) if (start is None or day.date() >= start.date()) and (end is None or day.date() <= end.date()): name = m.group(1) days[name.split("_", 1)[1] if "_" in name else name] += 1 if days: print(f" System metrics (metrics/, one CSV per metric and per day; read by metrics.py): {len(days)}: " + ", ".join(f"{k} ({n} d)" for k, n in sorted(days.items()))) for title, paths in (("Camera images", f["images"]), ("Differential images (images_diff/; no tool reads them)", glob.glob(os.path.join(folder, "images_diff", "*")))): times = [h for h in map(image_time, paths) if h and filter_window([{"t": h}], start, end)] if paths: print(f" {title}: {len(times)}") if others: print(" Others: " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(others.items()))) def images(paths, start=None, end=None): times = sorted(h for h in map(image_time, paths) if h) print(f"Images: {len(paths)}" + (f", from {iso(times[0])} to {iso(times[-1])}" if times else "")) unknown = [p for p in paths if image_time(p) is None] if unknown: print(f" names without a recognised time: {len(unknown)}") times = [h for h in times if filter_window([{"t": h}], start, end)] if len(times) < 2: return # Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot): # one line per change of gap, without losing any of the times covered. runs = [] # [first, last, gap in min, number of gaps] for a, b in zip(times, times[1:]): e = round((b - a).total_seconds() / 300) * 5 if runs and runs[-1][2] == e: runs[-1][1], runs[-1][3] = b, runs[-1][3] + 1 else: runs.append([a, b, e, 1]) what = "over the window" if start or end else "between the first and the last" print(f" gap between two images {what}, per run of equal gaps (within 5 min):") for a, b, e, n in runs: print(f" {iso(a)} → {iso(b)}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min")) def measurements(by): if not by: print('\nNo measurement: no "advanced" export in this folder.') return instants = sorted({m["t"] for ms in by.values() for m in ms}) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants") print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min") gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])] gaps = [x for x in gaps if x[2] > 0] print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}") for a, b, n in gaps: print(f" between {iso(a)} and {iso(b)}: {n}") names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) refs = [s for s in names if is_reference(by[s])] print(f"\nSensors: {len(by)} ({len(refs)} references, {len(by) - len(refs)} targets)") empty = [s for s in names if not any(m["pattern_type"] for m in by[s])] if empty: print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled " "(references only, labeling/knowledge/displacement-columns.md)") print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}") for s in names: ms = by[s] threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms})) print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} " f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8}") def camera(folder, by, start=None, end=None): """Camera poses, and the measurement instants without a pose (the camera was not registered).""" poses = filter_window(load_camera(folder), start, end) instants = sorted({m["t"] for ms in by.values() for m in filter_window(ms, start, end)}) what = "in the window" if start or end else "over the whole period" if not poses: print(f"\nCamera pose {what}: none") return with_pose = {p["t"].replace(second=0, microsecond=0) for p in poses} print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}") runs = [] # [start, end, count, rank] of the consecutive measurement instants without a pose for i, t in enumerate(instants): if t.replace(second=0, microsecond=0) in with_pose: continue if runs and runs[-1][3] == i - 1: runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i else: runs.append([t, t, 1, i]) print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}") if start or end: for a, b, n, _ in runs: print(f" {span(a, b, n)}") def window(by, start, end): """Absent steps and rejections dated over the start/end window.""" instants = sorted({m["t"] for ms in by.values() for m in ms}) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) series = {s: {m["t"]: m for m in filter_window(ms, start, end)} for s, ms in by.items()} inside = sorted({t for d in series.values() for t in d}) print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present") names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):") gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(inside, inside[1:])] gaps = [x for x in gaps if x[2] > 0] for a, b, n in gaps: print(f" between {iso(a)} and {iso(b)}: {n}") if not gaps: print(" (none)") print("Present instants where sensors have no measurement:") partial = [(t, [s for s in names if t not in series[s]]) for t in inside] for t, lacking in partial: if lacking: print(f" {iso(t)}: {len(lacking)} sensors: {','.join(lacking)}") if not any(m for _, m in partial): print(" (none)") print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):") for s in names: rej = [t for t in sorted(series[s]) if rejected(series[s][t])] runs = [] for t in rej: if runs and t - runs[-1][1] <= cadence * 1.5: runs[-1][1], runs[-1][2] = t, runs[-1][2] + 1 else: runs.append([t, t, 1]) if rej: print(f" {s} ({len(rej)}): " + "; ".join(span(a, b, n) for a, b, n in runs)) if not any(rejected(m) for d in series.values() for m in d.values()): print(" (none)") refs = [s for s in names if is_reference(by[s])] # inlier says which references the compensation retains; some exports do not fill it (and sometimes # only reproj_err): filled reproj_err is taken then, and the title says so. col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None) if col is None: print("Retained references by the compensation: columns inlier and reproj_err absent or empty.") return retained = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None) def mark(s, t): m = series[s].get(t) return "absent" if m is None else "r" if rejected(m) else "empty" if m[col] is None else f"{m[col]:g}" out = [(t, tuple((s, mark(s, t)) for s in refs if t not in series[s] or not retained(series[s][t]))) for t in inside] count = collections.Counter(len(refs) - len(h) for _, h in out) what = "inlier = 1" if col == "inlier" else "inlier empty over the window: reproj_err filled instead" print(f"Retained references by the compensation ({what}) out of {len(refs)}, number of instants: " + ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True))) print("Instants where references are not retained (runs of consecutive steps with the same references out; " f"r = rejected, empty = {col} empty, otherwise the inlier value):") runs = [] for i, (t, h) in enumerate(out): if h and runs and runs[-1][3] == h and runs[-1][4] == i - 1: runs[-1][1], runs[-1][2], runs[-1][4] = t, runs[-1][2] + 1, i elif h: runs.append([t, t, 1, h, i]) for a, b, n, h, _ in runs: print(f" {span(a, b, n)}: {len(refs) - len(h)} retained; out: " + ", ".join(f"{s} ({q})" for s, q in h)) if not runs: print(" (none)") def main(folder, start=None, end=None, full=False): print(f"Folder: {folder}") f = find_files(folder) files(f, full) contents(folder, f, start, end) images(f["images"], start, end) by = load(folder) measurements(by) if by: camera(folder, by) if by and (start or end): window(by, start, end) camera(folder, by, start, end) if __name__ == "__main__": a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folder") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) a.add_argument("--files", action="store_true", help="full list of the files (otherwise: the first and the last beyond 5)") o = a.parse_args() main(o.folder, o.start, o.end, o.files)