"""Inventory of a station folder: what is available, what is missing. Usage: python3 tools/scripts/inventory.py data/PS_DES_COUDRES/ [--start T] [--end T] [--files] The counts cover the whole period. With --start and/or --end, one more section lists, over this window, the absent steps (dated, with the sensors that have no measurement) and, per sensor, the rejected measurements (dated, in runs of consecutive steps; rejected: error_value above the threshold, references included), then the number of references retained by the compensation (inlier = 1; inlier empty over the window: reproj_err filled) at each instant and, dated, the runs of instants where references are not (which ones, and why: rejected, inlier value, empty). Camera movements (camera/): number of poses, period, and the measurement instants without a pose (no registration), over the whole period then over the window. Before the counts, what each source contains (over the window, otherwise the whole period): the columns of the export, of the pose, of quality/ and of weather/, filled or empty (with, for an export column that not all sensors carry, how many carry it), the names of the system metrics of metrics/, the images and the differential images. Duplicate readings (same sensor, same instant in two exports, or twice in one) are counted per export. Images: per run of equal gaps between two images (within 5 min; over the window if it is given). A dated run that ends on the same day writes only the time of its end. Gives values, not a verdict. """ import argparse import collections import csv import glob import os import statistics from datetime import datetime, timezone from common import QUALITY, group, without_pose from data import (METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, load_targets, parse_time, rejected) def span(a, b, n): """A dated run: the start in full, the end without the date when it falls on the same day.""" if n == 1: return iso(a) return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})" def duplicates(paths): """Readings (same sensor, same instant) already read in a previous export or higher up in the same one: load keeps only one, that of the last file in name order. One line per export that has some.""" seen, lines = {}, [] for p in paths: n = already = different = 0 with open(p, newline="") as fh: for r in csv.DictReader(fh): n += 1 key = (r["sensor_name"].strip(), r["time_utc"]) val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None)) if key in seen: already += 1 different += seen[key] != val seen[key] = val if already: lines.append(f" {os.path.basename(p)}: {already} of {n} already read, {different} of them with different values") if lines: print("Duplicate readings (same sensor, same instant in an export already read, in name order; " "the tools keep the one of the last file):") print("\n".join(lines)) def list_paths(paths, full): """A pumping folder holds hundreds of files: beyond 5, the first and the last only.""" if full or len(paths) <= 5: shown = paths else: shown = [paths[0], paths[-1]] print(f" {len(paths)} files, first and last (--files for the full list):") for p in shown: print(f" {p}") def files(f, full=False): print(f'"advanced" exports (source of the measurements): {len(f["advanced"])}') list_paths(f["advanced"], full) duplicates(f["advanced"]) print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, " "local time without a zone; see tools/DATA.md)") list_paths(f["hourly"], full) print(f"Camera movements (pose at each registration): {len(f['camera'])}") list_paths(f["camera"], full) print(f"Other files: {len(f['others'])}") list_paths(f["others"], full) def filled(paths, start, end): """CSVs of a source -> (rows of the window, {column: rows with a value}, {column: sensors that have one}), columns in header order; sensors: sensor_name, if there is one.""" rows, count, who = 0, {}, collections.defaultdict(set) for p in paths: with open(p, newline="") as fh: r = csv.DictReader(fh) for c in r.fieldnames or []: if c not in ("", "time_utc", "sensor_name"): count.setdefault(c, 0) for row in r: if (start or end) and filter_window([{"t": parse_time(row["time_utc"])}], start, end) == []: continue rows += 1 for c in count: if (row.get(c) or "").strip(): count[c] += 1 if row.get("sensor_name"): who[c].add(row["sensor_name"].strip()) return rows, count, who def contents(folder, f, start=None, end=None): """What each source contains over the window: its columns, filled or empty, and the names of the metrics. Names and counts, never values (metrics/ carries identifiers: SIM_CCID, ip_*).""" what = "over the window" if start or end else "over the whole period" print(f'\nWhat the data contain, {what} (per column, the rows that have a value; "empty": none):') others = collections.defaultdict(list) for p in f["others"]: others[os.path.relpath(p, folder).split(os.sep)[0]].append(p) for title, paths in (('"advanced" export', f["advanced"]), ("Camera pose (camera/)", f["camera"]), ("Image quality (quality/)", others.pop("quality", [])), ("Weather (weather/, nearest MeteoSwiss station, hourly; tools/DATA.md, no tool plots it)", others.pop("weather", []))): if paths: n, count, who = filled(paths, start, end) everyone = set().union(*who.values()) def carriers(c): # number of sensors that carry the column, when it is not all of them return f" ({len(who[c])} sensors of {len(everyone)})" if everyone and len(who[c]) < len(everyone) else "" print(f" {title}: {n} rows; " + ", ".join(f"{c} {k}{carriers(c)}" if k else f"{c} empty" for c, k in count.items())) if paths is f["advanced"]: q = [c for c in QUALITY if count.get(c)] print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, " "tools/DATA.md" if q else "empty")) days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window for p in others.pop("metrics", []): m = METRIC_RE.match(os.path.basename(p)) if not m: continue day = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc) if (start is None or day.date() >= start.date()) and (end is None or day.date() <= end.date()): name = m.group(1) days[name.split("_", 1)[1] if "_" in name else name] += 1 if days: print(f" System metrics (metrics/, one CSV per metric and per day; read by metrics.py): {len(days)}: " + ", ".join(f"{k} ({n} d)" for k, n in sorted(days.items()))) for title, paths in (("Camera images", f["images"]), ("Differential images (images_diff/; no tool reads them)", glob.glob(os.path.join(folder, "images_diff", "*")))): times = [h for h in map(image_time, paths) if h and filter_window([{"t": h}], start, end)] if paths: print(f" {title}: {len(times)}") if others: print(" Others: " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(others.items()))) def images(paths, start=None, end=None): times = sorted(h for h in map(image_time, paths) if h) print(f"Images: {len(paths)}" + (f", from {iso(times[0])} to {iso(times[-1])}" if times else "")) unknown = [p for p in paths if image_time(p) is None] if unknown: print(f" names without a recognised time: {len(unknown)}") times = [h for h in times if filter_window([{"t": h}], start, end)] if len(times) < 2: return # Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot): # one line per change of gap, without losing any of the times covered. pairs = [(a, b, round((b - a).total_seconds() / 300) * 5) for a, b in zip(times, times[1:])] what = "over the window" if start or end else "between the first and the last" print(f" gap between two images {what}, per run of equal gaps (within 5 min):") for g in group(pairs, lambda p, q: p[2] == q[2]): n, e = len(g), g[0][2] print(f" {iso(g[0][0])} → {iso(g[-1][1])}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min")) def missing(instants, cadence): """Gaps between two present instants: [(before, after, number of absent steps)].""" return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0] def measurements(by, targets=None): if not by: print('\nNo measurement: no "advanced" export in this folder.') return instants = sorted({m["t"] for ms in by.values() for m in ms}) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants") print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min") gaps = missing(instants, cadence) print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}") for a, b, n in gaps: print(f" between {iso(a)} and {iso(b)}: {n}") names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) refs = [s for s in names if is_reference(by[s])] print(f"\nSensors: {len(by)} ({len(refs)} references, {len(by) - len(refs)} targets)") empty = [s for s in names if not any(m["pattern_type"] for m in by[s])] if empty: print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled " "(references only, tools/DATA.md)") targets = targets or {} if targets: print(" distance (m) and mm/px: targets.csv (data.load_targets); \"-\": not in it, or not filled in " "(distance 1000 mm); \"?\": its lines (n) disagree, no corr_x_mm / corr_y_mm") else: print(" no targets.csv: no distance, no corr_x_mm / corr_y_mm") print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}" f" {'distance':>9} {'mm/px':>8}") for s in names: ms = by[s] threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms})) t = targets.get(s, {}) mark = f"?({t['lines']})" if t.get("lines", 0) > 1 else "-" distance = mark if t.get("distance_mm") is None else f"{t['distance_mm'] / 1000:g}" factor = mark if t.get("mm_per_px") is None else f"{t['mm_per_px']:.3f}" print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} " f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8} {distance:>9} {factor:>8}") def camera(folder, by, start=None, end=None): """Camera poses, and the measurement instants without a pose (the camera was not registered).""" poses = filter_window(load_camera(folder), start, end) instants = sorted({m["t"] for ms in by.values() for m in filter_window(ms, start, end)}) what = "in the window" if start or end else "over the whole period" if not poses: print(f"\nCamera pose {what}: none") return print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}") runs = without_pose(instants, poses) print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}") if start or end: for a, b, n in runs: print(f" {span(a, b, n)}") def window(by, start, end): """Absent steps and rejections dated over the start/end window.""" instants = sorted({m["t"] for ms in by.values() for m in ms}) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) series = {s: {m["t"]: m for m in filter_window(ms, start, end)} for s, ms in by.items()} inside = sorted({t for d in series.values() for t in d}) print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present") names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):") gaps = missing(inside, cadence) for a, b, n in gaps: print(f" between {iso(a)} and {iso(b)}: {n}") if not gaps: print(" (none)") print("Present instants where sensors have no measurement:") partial = [(t, [s for s in names if t not in series[s]]) for t in inside] for t, lacking in partial: if lacking: print(f" {iso(t)}: {len(lacking)} sensors: {','.join(lacking)}") if not any(m for _, m in partial): print(" (none)") print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):") for s in names: rej = [t for t in sorted(series[s]) if rejected(series[s][t])] if rej: print(f" {s} ({len(rej)}): " + "; ".join(span(g[0], g[-1], len(g)) for g in group(rej, lambda a, b: b - a <= cadence * 1.5))) if not any(rejected(m) for d in series.values() for m in d.values()): print(" (none)") refs = [s for s in names if is_reference(by[s])] # inlier says which references the compensation retains; some exports do not fill it (and sometimes # only reproj_err): filled reproj_err is taken then, and the title says so. col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None) if col is None: print("Retained references by the compensation: columns inlier and reproj_err absent or empty.") return retained = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None) def mark(s, t): m = series[s].get(t) return "absent" if m is None else "r" if rejected(m) else "empty" if m[col] is None else f"{m[col]:g}" out = [(t, tuple((s, mark(s, t)) for s in refs if t not in series[s] or not retained(series[s][t]))) for t in inside] count = collections.Counter(len(refs) - len(h) for _, h in out) what = "inlier = 1" if col == "inlier" else "inlier empty over the window: reproj_err filled instead" print(f"Retained references by the compensation ({what}) out of {len(refs)}, number of instants: " + ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True))) print("Instants where references are not retained (runs of consecutive steps with the same references out; " f"r = rejected, empty = {col} empty, otherwise the inlier value):") runs = group([(i, t, h) for i, (t, h) in enumerate(out) if h], lambda a, b: b[0] == a[0] + 1 and b[2] == a[2]) for g in runs: h = g[0][2] print(f" {span(g[0][1], g[-1][1], len(g))}: {len(refs) - len(h)} retained; out: " + ", ".join(f"{s} ({q})" for s, q in h)) if not runs: print(" (none)") def main(folder, start=None, end=None, full=False): print(f"Folder: {folder}") f = find_files(folder) files(f, full) contents(folder, f, start, end) images(f["images"], start, end) by = load(folder) measurements(by, load_targets(folder)) if by: camera(folder, by) if by and (start or end): window(by, start, end) camera(folder, by, start, end) if __name__ == "__main__": a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folder") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) a.add_argument("--files", action="store_true", help="full list of the files (otherwise: the first and the last beyond 5)") o = a.parse_args() main(o.folder, o.start, o.end, o.files)