2026-11-16 12:40:03 +01:00
|
|
|
"""Inventory of a station folder: what is available, what is missing.
|
|
|
|
|
|
|
|
|
|
Usage: python3 tools/scripts/inventory.py data/PS_DES_COUDRES/ [--start T] [--end T] [--files]
|
|
|
|
|
The counts cover the whole period. With --start and/or --end, one more section lists, over this window,
|
|
|
|
|
the absent steps (dated, with the sensors that have no measurement) and, per sensor, the rejected measurements (dated,
|
|
|
|
|
in runs of consecutive steps; rejected: error_value above the threshold, references included), then the number of
|
|
|
|
|
references retained by the compensation (inlier = 1; inlier empty over the window: reproj_err filled) at each instant
|
|
|
|
|
and, dated, the runs of instants where references are not (which ones, and why: rejected, inlier value,
|
|
|
|
|
empty). Camera movements (camera/): number of poses, period, and the measurement instants
|
|
|
|
|
without a pose (no registration), over the whole period then over the window. Before the counts, what each source
|
|
|
|
|
contains (over the window, otherwise the whole period): the columns of the export, of the pose and of quality/, filled or empty (with,
|
|
|
|
|
for an export column that not all sensors carry, how many carry it), the names of the system metrics
|
|
|
|
|
of metrics/, the images and the differential images. Duplicate readings (same sensor, same instant in two
|
|
|
|
|
exports, or twice in one) are counted per export. Images: per run of equal gaps between two images (within 5 min;
|
|
|
|
|
over the window if it is given). A dated run that ends on the same day writes only the time of its end.
|
|
|
|
|
Gives values, not a verdict.
|
|
|
|
|
"""
|
|
|
|
|
import argparse
|
|
|
|
|
import collections
|
|
|
|
|
import csv
|
|
|
|
|
import glob
|
|
|
|
|
import os
|
|
|
|
|
import statistics
|
2026-11-16 15:48:42 +01:00
|
|
|
from datetime import datetime, timezone
|
2026-11-16 12:40:03 +01:00
|
|
|
|
2026-11-16 15:48:42 +01:00
|
|
|
from common import QUALITY, group, without_pose
|
|
|
|
|
from data import METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
|
2026-11-16 12:40:03 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def span(a, b, n):
|
|
|
|
|
"""A dated run: the start in full, the end without the date when it falls on the same day."""
|
|
|
|
|
if n == 1:
|
|
|
|
|
return iso(a)
|
|
|
|
|
return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def duplicates(paths):
|
|
|
|
|
"""Readings (same sensor, same instant) already read in a previous export or higher up in the same one: load
|
|
|
|
|
keeps only one, that of the last file in name order. One line per export that has some."""
|
|
|
|
|
seen, lines = {}, []
|
|
|
|
|
for p in paths:
|
|
|
|
|
n = already = different = 0
|
|
|
|
|
with open(p, newline="") as fh:
|
|
|
|
|
for r in csv.DictReader(fh):
|
|
|
|
|
n += 1
|
|
|
|
|
key = (r["sensor_name"].strip(), r["time_utc"])
|
|
|
|
|
val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None))
|
|
|
|
|
if key in seen:
|
|
|
|
|
already += 1
|
|
|
|
|
different += seen[key] != val
|
|
|
|
|
seen[key] = val
|
|
|
|
|
if already:
|
|
|
|
|
lines.append(f" {os.path.basename(p)}: {already} of {n} already read, {different} of them with different values")
|
|
|
|
|
if lines:
|
|
|
|
|
print("Duplicate readings (same sensor, same instant in an export already read, in name order; "
|
|
|
|
|
"the tools keep the one of the last file):")
|
|
|
|
|
print("\n".join(lines))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def list_paths(paths, full):
|
|
|
|
|
"""A pumping folder holds hundreds of files: beyond 5, the first and the last only."""
|
|
|
|
|
if full or len(paths) <= 5:
|
|
|
|
|
shown = paths
|
|
|
|
|
else:
|
|
|
|
|
shown = [paths[0], paths[-1]]
|
|
|
|
|
print(f" {len(paths)} files, first and last (--files for the full list):")
|
|
|
|
|
for p in shown:
|
|
|
|
|
print(f" {p}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def files(f, full=False):
|
|
|
|
|
print(f'"advanced" exports (source of the measurements): {len(f["advanced"])}')
|
|
|
|
|
list_paths(f["advanced"], full)
|
|
|
|
|
duplicates(f["advanced"])
|
|
|
|
|
print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, "
|
2026-11-16 12:41:44 +01:00
|
|
|
"local time without a zone; see tools/DATA.md)")
|
2026-11-16 12:40:03 +01:00
|
|
|
list_paths(f["hourly"], full)
|
|
|
|
|
print(f"Camera movements (pose at each registration): {len(f['camera'])}")
|
|
|
|
|
list_paths(f["camera"], full)
|
|
|
|
|
print(f"Other files: {len(f['others'])}")
|
|
|
|
|
list_paths(f["others"], full)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def filled(paths, start, end):
|
|
|
|
|
"""CSVs of a source -> (rows of the window, {column: rows with a value}, {column: sensors that have one}),
|
|
|
|
|
columns in header order; sensors: sensor_name, if there is one."""
|
|
|
|
|
rows, count, who = 0, {}, collections.defaultdict(set)
|
|
|
|
|
for p in paths:
|
|
|
|
|
with open(p, newline="") as fh:
|
|
|
|
|
r = csv.DictReader(fh)
|
|
|
|
|
for c in r.fieldnames or []:
|
|
|
|
|
if c not in ("", "time_utc", "sensor_name"):
|
|
|
|
|
count.setdefault(c, 0)
|
|
|
|
|
for row in r:
|
|
|
|
|
if (start or end) and filter_window([{"t": parse_time(row["time_utc"])}], start, end) == []:
|
|
|
|
|
continue
|
|
|
|
|
rows += 1
|
|
|
|
|
for c in count:
|
|
|
|
|
if (row.get(c) or "").strip():
|
|
|
|
|
count[c] += 1
|
|
|
|
|
if row.get("sensor_name"):
|
|
|
|
|
who[c].add(row["sensor_name"].strip())
|
|
|
|
|
return rows, count, who
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def contents(folder, f, start=None, end=None):
|
|
|
|
|
"""What each source contains over the window: its columns, filled or empty, and the names of the metrics.
|
|
|
|
|
Names and counts, never values (metrics/ carries identifiers: SIM_CCID, ip_*)."""
|
|
|
|
|
what = "over the window" if start or end else "over the whole period"
|
|
|
|
|
print(f'\nWhat the data contain, {what} (per column, the rows that have a value; "empty": none):')
|
|
|
|
|
others = collections.defaultdict(list)
|
|
|
|
|
for p in f["others"]:
|
|
|
|
|
others[os.path.relpath(p, folder).split(os.sep)[0]].append(p)
|
|
|
|
|
for title, paths in (('"advanced" export', f["advanced"]), ("Camera pose (camera/)", f["camera"]),
|
|
|
|
|
("Image quality (quality/)", others.pop("quality", []))):
|
|
|
|
|
if paths:
|
|
|
|
|
n, count, who = filled(paths, start, end)
|
|
|
|
|
everyone = set().union(*who.values())
|
|
|
|
|
def carriers(c): # number of sensors that carry the column, when it is not all of them
|
|
|
|
|
return f" ({len(who[c])} sensors of {len(everyone)})" if everyone and len(who[c]) < len(everyone) else ""
|
|
|
|
|
print(f" {title}: {n} rows; " + ", ".join(f"{c} {k}{carriers(c)}" if k else f"{c} empty" for c, k in count.items()))
|
|
|
|
|
if paths is f["advanced"]:
|
|
|
|
|
q = [c for c in QUALITY if count.get(c)]
|
|
|
|
|
print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, "
|
2026-11-16 12:41:44 +01:00
|
|
|
"tools/DATA.md" if q else "empty"))
|
2026-11-16 12:40:03 +01:00
|
|
|
days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window
|
|
|
|
|
for p in others.pop("metrics", []):
|
|
|
|
|
m = METRIC_RE.match(os.path.basename(p))
|
|
|
|
|
if not m:
|
|
|
|
|
continue
|
|
|
|
|
day = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc)
|
|
|
|
|
if (start is None or day.date() >= start.date()) and (end is None or day.date() <= end.date()):
|
|
|
|
|
name = m.group(1)
|
|
|
|
|
days[name.split("_", 1)[1] if "_" in name else name] += 1
|
|
|
|
|
if days:
|
|
|
|
|
print(f" System metrics (metrics/, one CSV per metric and per day; read by metrics.py): {len(days)}: "
|
|
|
|
|
+ ", ".join(f"{k} ({n} d)" for k, n in sorted(days.items())))
|
|
|
|
|
for title, paths in (("Camera images", f["images"]),
|
|
|
|
|
("Differential images (images_diff/; no tool reads them)", glob.glob(os.path.join(folder, "images_diff", "*")))):
|
|
|
|
|
times = [h for h in map(image_time, paths) if h and filter_window([{"t": h}], start, end)]
|
|
|
|
|
if paths:
|
|
|
|
|
print(f" {title}: {len(times)}")
|
|
|
|
|
if others:
|
|
|
|
|
print(" Others: " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(others.items())))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def images(paths, start=None, end=None):
|
|
|
|
|
times = sorted(h for h in map(image_time, paths) if h)
|
|
|
|
|
print(f"Images: {len(paths)}" + (f", from {iso(times[0])} to {iso(times[-1])}" if times else ""))
|
|
|
|
|
unknown = [p for p in paths if image_time(p) is None]
|
|
|
|
|
if unknown:
|
|
|
|
|
print(f" names without a recognised time: {len(unknown)}")
|
|
|
|
|
times = [h for h in times if filter_window([{"t": h}], start, end)]
|
|
|
|
|
if len(times) < 2:
|
|
|
|
|
return
|
|
|
|
|
# Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot):
|
|
|
|
|
# one line per change of gap, without losing any of the times covered.
|
2026-11-16 15:48:42 +01:00
|
|
|
pairs = [(a, b, round((b - a).total_seconds() / 300) * 5) for a, b in zip(times, times[1:])]
|
2026-11-16 12:40:03 +01:00
|
|
|
what = "over the window" if start or end else "between the first and the last"
|
|
|
|
|
print(f" gap between two images {what}, per run of equal gaps (within 5 min):")
|
2026-11-16 15:48:42 +01:00
|
|
|
for g in group(pairs, lambda p, q: p[2] == q[2]):
|
|
|
|
|
n, e = len(g), g[0][2]
|
|
|
|
|
print(f" {iso(g[0][0])} → {iso(g[-1][1])}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def missing(instants, cadence):
|
|
|
|
|
"""Gaps between two present instants: [(before, after, number of absent steps)]."""
|
|
|
|
|
return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0]
|
2026-11-16 12:40:03 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def measurements(by):
|
|
|
|
|
if not by:
|
|
|
|
|
print('\nNo measurement: no "advanced" export in this folder.')
|
|
|
|
|
return
|
|
|
|
|
instants = sorted({m["t"] for ms in by.values() for m in ms})
|
|
|
|
|
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
|
|
|
|
|
print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
|
|
|
|
|
print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min")
|
2026-11-16 15:48:42 +01:00
|
|
|
gaps = missing(instants, cadence)
|
2026-11-16 12:40:03 +01:00
|
|
|
print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}")
|
|
|
|
|
for a, b, n in gaps:
|
|
|
|
|
print(f" between {iso(a)} and {iso(b)}: {n}")
|
|
|
|
|
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
|
|
|
|
refs = [s for s in names if is_reference(by[s])]
|
|
|
|
|
print(f"\nSensors: {len(by)} ({len(refs)} references, {len(by) - len(refs)} targets)")
|
|
|
|
|
empty = [s for s in names if not any(m["pattern_type"] for m in by[s])]
|
|
|
|
|
if empty:
|
|
|
|
|
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
|
2026-11-16 12:41:44 +01:00
|
|
|
"(references only, tools/DATA.md)")
|
2026-11-16 12:40:03 +01:00
|
|
|
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
|
|
|
|
|
for s in names:
|
|
|
|
|
ms = by[s]
|
|
|
|
|
threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
|
|
|
|
|
print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} "
|
|
|
|
|
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def camera(folder, by, start=None, end=None):
|
|
|
|
|
"""Camera poses, and the measurement instants without a pose (the camera was not registered)."""
|
|
|
|
|
poses = filter_window(load_camera(folder), start, end)
|
|
|
|
|
instants = sorted({m["t"] for ms in by.values() for m in filter_window(ms, start, end)})
|
|
|
|
|
what = "in the window" if start or end else "over the whole period"
|
|
|
|
|
if not poses:
|
|
|
|
|
print(f"\nCamera pose {what}: none")
|
|
|
|
|
return
|
|
|
|
|
print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
|
2026-11-16 15:48:42 +01:00
|
|
|
runs = without_pose(instants, poses)
|
2026-11-16 12:40:03 +01:00
|
|
|
print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}")
|
|
|
|
|
if start or end:
|
2026-11-16 15:48:42 +01:00
|
|
|
for a, b, n in runs:
|
2026-11-16 12:40:03 +01:00
|
|
|
print(f" {span(a, b, n)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def window(by, start, end):
|
|
|
|
|
"""Absent steps and rejections dated over the start/end window."""
|
|
|
|
|
instants = sorted({m["t"] for ms in by.values() for m in ms})
|
|
|
|
|
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
|
|
|
|
|
series = {s: {m["t"]: m for m in filter_window(ms, start, end)} for s, ms in by.items()}
|
|
|
|
|
inside = sorted({t for d in series.values() for t in d})
|
|
|
|
|
print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present")
|
|
|
|
|
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
|
|
|
|
print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):")
|
2026-11-16 15:48:42 +01:00
|
|
|
gaps = missing(inside, cadence)
|
2026-11-16 12:40:03 +01:00
|
|
|
for a, b, n in gaps:
|
|
|
|
|
print(f" between {iso(a)} and {iso(b)}: {n}")
|
|
|
|
|
if not gaps:
|
|
|
|
|
print(" (none)")
|
|
|
|
|
print("Present instants where sensors have no measurement:")
|
|
|
|
|
partial = [(t, [s for s in names if t not in series[s]]) for t in inside]
|
|
|
|
|
for t, lacking in partial:
|
|
|
|
|
if lacking:
|
|
|
|
|
print(f" {iso(t)}: {len(lacking)} sensors: {','.join(lacking)}")
|
|
|
|
|
if not any(m for _, m in partial):
|
|
|
|
|
print(" (none)")
|
|
|
|
|
print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):")
|
|
|
|
|
for s in names:
|
|
|
|
|
rej = [t for t in sorted(series[s]) if rejected(series[s][t])]
|
|
|
|
|
if rej:
|
2026-11-16 15:48:42 +01:00
|
|
|
print(f" {s} ({len(rej)}): " + "; ".join(span(g[0], g[-1], len(g)) for g in group(rej, lambda a, b: b - a <= cadence * 1.5)))
|
2026-11-16 12:40:03 +01:00
|
|
|
if not any(rejected(m) for d in series.values() for m in d.values()):
|
|
|
|
|
print(" (none)")
|
|
|
|
|
refs = [s for s in names if is_reference(by[s])]
|
|
|
|
|
# inlier says which references the compensation retains; some exports do not fill it (and sometimes
|
|
|
|
|
# only reproj_err): filled reproj_err is taken then, and the title says so.
|
|
|
|
|
col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None)
|
|
|
|
|
if col is None:
|
|
|
|
|
print("Retained references by the compensation: columns inlier and reproj_err absent or empty.")
|
|
|
|
|
return
|
|
|
|
|
retained = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None)
|
|
|
|
|
def mark(s, t):
|
|
|
|
|
m = series[s].get(t)
|
|
|
|
|
return "absent" if m is None else "r" if rejected(m) else "empty" if m[col] is None else f"{m[col]:g}"
|
|
|
|
|
out = [(t, tuple((s, mark(s, t)) for s in refs if t not in series[s] or not retained(series[s][t]))) for t in inside]
|
|
|
|
|
count = collections.Counter(len(refs) - len(h) for _, h in out)
|
|
|
|
|
what = "inlier = 1" if col == "inlier" else "inlier empty over the window: reproj_err filled instead"
|
|
|
|
|
print(f"Retained references by the compensation ({what}) out of {len(refs)}, number of instants: "
|
|
|
|
|
+ ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True)))
|
|
|
|
|
print("Instants where references are not retained (runs of consecutive steps with the same references out; "
|
|
|
|
|
f"r = rejected, empty = {col} empty, otherwise the inlier value):")
|
2026-11-16 15:48:42 +01:00
|
|
|
runs = group([(i, t, h) for i, (t, h) in enumerate(out) if h], lambda a, b: b[0] == a[0] + 1 and b[2] == a[2])
|
|
|
|
|
for g in runs:
|
|
|
|
|
h = g[0][2]
|
|
|
|
|
print(f" {span(g[0][1], g[-1][1], len(g))}: {len(refs) - len(h)} retained; out: "
|
2026-11-16 12:40:03 +01:00
|
|
|
+ ", ".join(f"{s} ({q})" for s, q in h))
|
|
|
|
|
if not runs:
|
|
|
|
|
print(" (none)")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(folder, start=None, end=None, full=False):
|
|
|
|
|
print(f"Folder: {folder}")
|
|
|
|
|
f = find_files(folder)
|
|
|
|
|
files(f, full)
|
|
|
|
|
contents(folder, f, start, end)
|
|
|
|
|
images(f["images"], start, end)
|
|
|
|
|
by = load(folder)
|
|
|
|
|
measurements(by)
|
|
|
|
|
if by:
|
|
|
|
|
camera(folder, by)
|
|
|
|
|
if by and (start or end):
|
|
|
|
|
window(by, start, end)
|
|
|
|
|
camera(folder, by, start, end)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
|
|
|
a.add_argument("folder")
|
|
|
|
|
a.add_argument("--start", type=parse_time)
|
|
|
|
|
a.add_argument("--end", type=parse_time)
|
|
|
|
|
a.add_argument("--files", action="store_true", help="full list of the files (otherwise: the first and the last beyond 5)")
|
|
|
|
|
o = a.parse_args()
|
|
|
|
|
main(o.folder, o.start, o.end, o.files)
|