Files
aimsight-tools/scripts/inventory.py
T

314 lines
16 KiB
Python
Raw Normal View History

"""Inventory of a station folder: what is available, what is missing.
Usage: python3 tools/scripts/inventory.py data/PS_DES_COUDRES/ [--start T] [--end T] [--files]
The counts cover the whole period. With --start and/or --end, one more section lists, over this window,
the absent steps (dated, with the sensors that have no measurement) and, per sensor, the rejected measurements (dated,
in runs of consecutive steps; rejected: error_value above the threshold, references included), then the number of
references retained by the compensation (inlier = 1; inlier empty over the window: reproj_err filled) at each instant
and, dated, the runs of instants where references are not (which ones, and why: rejected, inlier value,
empty). Camera movements (camera/): number of poses, period, and the measurement instants
without a pose (no registration), over the whole period then over the window. Before the counts, what each source
contains (over the window, otherwise the whole period): the columns of the export, of the pose and of quality/, filled or empty (with,
for an export column that not all sensors carry, how many carry it), the names of the system metrics
of metrics/, the images and the differential images. Duplicate readings (same sensor, same instant in two
exports, or twice in one) are counted per export. Images: per run of equal gaps between two images (within 5 min;
over the window if it is given). A dated run that ends on the same day writes only the time of its end.
Gives values, not a verdict.
"""
import argparse
import collections
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timedelta, timezone
from data import filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
from summary import QUALITY
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_\d{2}_\d{2}") # NIEDERHORN_BusCurrent_2026_09_22_00_00 to ….csv
def span(a, b, n):
"""A dated run: the start in full, the end without the date when it falls on the same day."""
if n == 1:
return iso(a)
return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})"
def duplicates(paths):
"""Readings (same sensor, same instant) already read in a previous export or higher up in the same one: load
keeps only one, that of the last file in name order. One line per export that has some."""
seen, lines = {}, []
for p in paths:
n = already = different = 0
with open(p, newline="") as fh:
for r in csv.DictReader(fh):
n += 1
key = (r["sensor_name"].strip(), r["time_utc"])
val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None))
if key in seen:
already += 1
different += seen[key] != val
seen[key] = val
if already:
lines.append(f" {os.path.basename(p)}: {already} of {n} already read, {different} of them with different values")
if lines:
print("Duplicate readings (same sensor, same instant in an export already read, in name order; "
"the tools keep the one of the last file):")
print("\n".join(lines))
def list_paths(paths, full):
"""A pumping folder holds hundreds of files: beyond 5, the first and the last only."""
if full or len(paths) <= 5:
shown = paths
else:
shown = [paths[0], paths[-1]]
print(f" {len(paths)} files, first and last (--files for the full list):")
for p in shown:
print(f" {p}")
def files(f, full=False):
print(f'"advanced" exports (source of the measurements): {len(f["advanced"])}')
list_paths(f["advanced"], full)
duplicates(f["advanced"])
print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, "
"local time without a zone; see tools/DATA.md)")
list_paths(f["hourly"], full)
print(f"Camera movements (pose at each registration): {len(f['camera'])}")
list_paths(f["camera"], full)
print(f"Other files: {len(f['others'])}")
list_paths(f["others"], full)
def filled(paths, start, end):
"""CSVs of a source -> (rows of the window, {column: rows with a value}, {column: sensors that have one}),
columns in header order; sensors: sensor_name, if there is one."""
rows, count, who = 0, {}, collections.defaultdict(set)
for p in paths:
with open(p, newline="") as fh:
r = csv.DictReader(fh)
for c in r.fieldnames or []:
if c not in ("", "time_utc", "sensor_name"):
count.setdefault(c, 0)
for row in r:
if (start or end) and filter_window([{"t": parse_time(row["time_utc"])}], start, end) == []:
continue
rows += 1
for c in count:
if (row.get(c) or "").strip():
count[c] += 1
if row.get("sensor_name"):
who[c].add(row["sensor_name"].strip())
return rows, count, who
def contents(folder, f, start=None, end=None):
"""What each source contains over the window: its columns, filled or empty, and the names of the metrics.
Names and counts, never values (metrics/ carries identifiers: SIM_CCID, ip_*)."""
what = "over the window" if start or end else "over the whole period"
print(f'\nWhat the data contain, {what} (per column, the rows that have a value; "empty": none):')
others = collections.defaultdict(list)
for p in f["others"]:
others[os.path.relpath(p, folder).split(os.sep)[0]].append(p)
for title, paths in (('"advanced" export', f["advanced"]), ("Camera pose (camera/)", f["camera"]),
("Image quality (quality/)", others.pop("quality", []))):
if paths:
n, count, who = filled(paths, start, end)
everyone = set().union(*who.values())
def carriers(c): # number of sensors that carry the column, when it is not all of them
return f" ({len(who[c])} sensors of {len(everyone)})" if everyone and len(who[c]) < len(everyone) else ""
print(f" {title}: {n} rows; " + ", ".join(f"{c} {k}{carriers(c)}" if k else f"{c} empty" for c, k in count.items()))
if paths is f["advanced"]:
q = [c for c in QUALITY if count.get(c)]
print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, "
"tools/DATA.md" if q else "empty"))
days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window
for p in others.pop("metrics", []):
m = METRIC_RE.match(os.path.basename(p))
if not m:
continue
day = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc)
if (start is None or day.date() >= start.date()) and (end is None or day.date() <= end.date()):
name = m.group(1)
days[name.split("_", 1)[1] if "_" in name else name] += 1
if days:
print(f" System metrics (metrics/, one CSV per metric and per day; read by metrics.py): {len(days)}: "
+ ", ".join(f"{k} ({n} d)" for k, n in sorted(days.items())))
for title, paths in (("Camera images", f["images"]),
("Differential images (images_diff/; no tool reads them)", glob.glob(os.path.join(folder, "images_diff", "*")))):
times = [h for h in map(image_time, paths) if h and filter_window([{"t": h}], start, end)]
if paths:
print(f" {title}: {len(times)}")
if others:
print(" Others: " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(others.items())))
def images(paths, start=None, end=None):
times = sorted(h for h in map(image_time, paths) if h)
print(f"Images: {len(paths)}" + (f", from {iso(times[0])} to {iso(times[-1])}" if times else ""))
unknown = [p for p in paths if image_time(p) is None]
if unknown:
print(f" names without a recognised time: {len(unknown)}")
times = [h for h in times if filter_window([{"t": h}], start, end)]
if len(times) < 2:
return
# Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot):
# one line per change of gap, without losing any of the times covered.
runs = [] # [first, last, gap in min, number of gaps]
for a, b in zip(times, times[1:]):
e = round((b - a).total_seconds() / 300) * 5
if runs and runs[-1][2] == e:
runs[-1][1], runs[-1][3] = b, runs[-1][3] + 1
else:
runs.append([a, b, e, 1])
what = "over the window" if start or end else "between the first and the last"
print(f" gap between two images {what}, per run of equal gaps (within 5 min):")
for a, b, e, n in runs:
print(f" {iso(a)} → {iso(b)}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min"))
def measurements(by):
if not by:
print('\nNo measurement: no "advanced" export in this folder.')
return
instants = sorted({m["t"] for ms in by.values() for m in ms})
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min")
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])]
gaps = [x for x in gaps if x[2] > 0]
print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}")
for a, b, n in gaps:
print(f" between {iso(a)} and {iso(b)}: {n}")
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
refs = [s for s in names if is_reference(by[s])]
print(f"\nSensors: {len(by)} ({len(refs)} references, {len(by) - len(refs)} targets)")
empty = [s for s in names if not any(m["pattern_type"] for m in by[s])]
if empty:
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
"(references only, tools/DATA.md)")
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
for s in names:
ms = by[s]
threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} "
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8}")
def camera(folder, by, start=None, end=None):
"""Camera poses, and the measurement instants without a pose (the camera was not registered)."""
poses = filter_window(load_camera(folder), start, end)
instants = sorted({m["t"] for ms in by.values() for m in filter_window(ms, start, end)})
what = "in the window" if start or end else "over the whole period"
if not poses:
print(f"\nCamera pose {what}: none")
return
with_pose = {p["t"].replace(second=0, microsecond=0) for p in poses}
print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
runs = [] # [start, end, count, rank] of the consecutive measurement instants without a pose
for i, t in enumerate(instants):
if t.replace(second=0, microsecond=0) in with_pose:
continue
if runs and runs[-1][3] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
else:
runs.append([t, t, 1, i])
print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}")
if start or end:
for a, b, n, _ in runs:
print(f" {span(a, b, n)}")
def window(by, start, end):
"""Absent steps and rejections dated over the start/end window."""
instants = sorted({m["t"] for ms in by.values() for m in ms})
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
series = {s: {m["t"]: m for m in filter_window(ms, start, end)} for s, ms in by.items()}
inside = sorted({t for d in series.values() for t in d})
print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present")
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):")
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(inside, inside[1:])]
gaps = [x for x in gaps if x[2] > 0]
for a, b, n in gaps:
print(f" between {iso(a)} and {iso(b)}: {n}")
if not gaps:
print(" (none)")
print("Present instants where sensors have no measurement:")
partial = [(t, [s for s in names if t not in series[s]]) for t in inside]
for t, lacking in partial:
if lacking:
print(f" {iso(t)}: {len(lacking)} sensors: {','.join(lacking)}")
if not any(m for _, m in partial):
print(" (none)")
print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):")
for s in names:
rej = [t for t in sorted(series[s]) if rejected(series[s][t])]
runs = []
for t in rej:
if runs and t - runs[-1][1] <= cadence * 1.5:
runs[-1][1], runs[-1][2] = t, runs[-1][2] + 1
else:
runs.append([t, t, 1])
if rej:
print(f" {s} ({len(rej)}): " + "; ".join(span(a, b, n) for a, b, n in runs))
if not any(rejected(m) for d in series.values() for m in d.values()):
print(" (none)")
refs = [s for s in names if is_reference(by[s])]
# inlier says which references the compensation retains; some exports do not fill it (and sometimes
# only reproj_err): filled reproj_err is taken then, and the title says so.
col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None)
if col is None:
print("Retained references by the compensation: columns inlier and reproj_err absent or empty.")
return
retained = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None)
def mark(s, t):
m = series[s].get(t)
return "absent" if m is None else "r" if rejected(m) else "empty" if m[col] is None else f"{m[col]:g}"
out = [(t, tuple((s, mark(s, t)) for s in refs if t not in series[s] or not retained(series[s][t]))) for t in inside]
count = collections.Counter(len(refs) - len(h) for _, h in out)
what = "inlier = 1" if col == "inlier" else "inlier empty over the window: reproj_err filled instead"
print(f"Retained references by the compensation ({what}) out of {len(refs)}, number of instants: "
+ ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True)))
print("Instants where references are not retained (runs of consecutive steps with the same references out; "
f"r = rejected, empty = {col} empty, otherwise the inlier value):")
runs = []
for i, (t, h) in enumerate(out):
if h and runs and runs[-1][3] == h and runs[-1][4] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][4] = t, runs[-1][2] + 1, i
elif h:
runs.append([t, t, 1, h, i])
for a, b, n, h, _ in runs:
print(f" {span(a, b, n)}: {len(refs) - len(h)} retained; out: "
+ ", ".join(f"{s} ({q})" for s, q in h))
if not runs:
print(" (none)")
def main(folder, start=None, end=None, full=False):
print(f"Folder: {folder}")
f = find_files(folder)
files(f, full)
contents(folder, f, start, end)
images(f["images"], start, end)
by = load(folder)
measurements(by)
if by:
camera(folder, by)
if by and (start or end):
window(by, start, end)
camera(folder, by, start, end)
if __name__ == "__main__":
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--files", action="store_true", help="full list of the files (otherwise: the first and the last beyond 5)")
o = a.parse_args()
main(o.folder, o.start, o.end, o.files)