"""Inventory of a station folder: what is available, what is missing. Usage: python3 tools/scripts/inventory.py data/PS_DES_COUDRES/ [--start T] [--end T] [--files] The counts cover the whole period. With --start and/or --end, one more section lists, over this window, the absent steps (dated, with the sensors that have no measurement) and, per sensor, the rejected measurements (out of its measurements in the window; dated, in runs of consecutive steps; then the share rejected per UTC day; rejected: error_value above the threshold, references included), then the number of references retained by the compensation (inlier = 1; inlier empty over the window: reproj_err filled) at each instant and, dated, the runs of instants where references are not (which ones, and why: rejected, inlier value, empty). Camera movements (camera/): number of poses, period, and the measurement instants without a pose (no registration), over the whole period then over the window. Before the counts, what each source contains (over the window, otherwise the whole period): the columns of the export, of the pose, of quality/ and of weather/, filled or empty (with, for an export column that not all sensors carry, how many carry it), the names of the system metrics of metrics/, the images and the differential images. Duplicate readings (same sensor, same instant in two exports, or twice in one) are counted per export. Images: per run of equal gaps between two images (within 5 min; over the window if it is given). A dated run that ends on the same day writes only the time of its end. Gives values, not a verdict. """ import argparse import collections import csv import glob import os import statistics from datetime import datetime, timezone from common import QUALITY, group, without_pose from data import (METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, load_targets, parse_time, rejected) def span(a, b, n): """A dated run: the start in full, the end without the date when it falls on the same day.""" if n == 1: return iso(a) return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})" def duplicates(paths): """Readings (same sensor, same instant) already read in a previous export or higher up in the same one: load keeps only one, that of the last file in name order. One line per export that has some.""" seen, lines = {}, [] for p in paths: n = already = different = 0 with open(p, newline="") as fh: for r in csv.DictReader(fh): n += 1 key = (r["sensor_name"].strip(), r["time_utc"]) val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None)) if key in seen: already += 1 different += seen[key] != val seen[key] = val if already: lines.append(f" {os.path.basename(p)}: {already} of {n} already read, {different} of them with different values") if lines: print("Duplicate readings (same sensor, same instant in an export already read, in name order; " "the tools keep the one of the last file):") print("\n".join(lines)) def list_paths(paths, full): """A pumping folder holds hundreds of files: beyond 5, the first and the last only.""" if full or len(paths) <= 5: shown = paths else: shown = [paths[0], paths[-1]] print(f" {len(paths)} files, first and last (--files for the full list):") for p in shown: print(f" {p}") def files(f, full=False): print(f'"advanced" exports (source of the measurements): {len(f["advanced"])}') list_paths(f["advanced"], full) duplicates(f["advanced"]) print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, " "local time without a zone; see tools/DATA.md)") list_paths(f["hourly"], full) print(f"Camera movements (pose at each registration): {len(f['camera'])}") list_paths(f["camera"], full) print(f"Other files: {len(f['others'])}") list_paths(f["others"], full) def filled(paths, start, end): """CSVs of a source -> (rows of the window, {column: rows with a value}, {column: sensors that have one}), columns in header order; sensors: sensor_name, if there is one.""" rows, count, who = 0, {}, collections.defaultdict(set) for p in paths: with open(p, newline="") as fh: r = csv.DictReader(fh) for c in r.fieldnames or []: if c not in ("", "time_utc", "sensor_name"): count.setdefault(c, 0) for row in r: if (start or end) and filter_window([{"t": parse_time(row["time_utc"])}], start, end) == []: continue rows += 1 for c in count: if (row.get(c) or "").strip(): count[c] += 1 if row.get("sensor_name"): who[c].add(row["sensor_name"].strip()) return rows, count, who def contents(folder, f, start=None, end=None): """What each source contains over the window: its columns, filled or empty, and the names of the metrics. Names and counts, never values (metrics/ carries identifiers: SIM_CCID, ip_*).""" what = "over the window" if start or end else "over the whole period" print(f'\nWhat the data contain, {what} (per column, the rows that have a value; "empty": none):') others = collections.defaultdict(list) for p in f["others"]: others[os.path.relpath(p, folder).split(os.sep)[0]].append(p) for title, paths in (('"advanced" export', f["advanced"]), ("Camera pose (camera/)", f["camera"]), ("Image quality (quality/)", others.pop("quality", [])), ("Weather (weather/, nearest MeteoSwiss station, hourly; tools/DATA.md; metrics.py --weather)", others.pop("weather", []))): if paths: n, count, who = filled(paths, start, end) everyone = set().union(*who.values()) def carriers(c): # number of sensors that carry the column, when it is not all of them return f" ({len(who[c])} sensors of {len(everyone)})" if everyone and len(who[c]) < len(everyone) else "" print(f" {title}: {n} rows; " + ", ".join(f"{c} {k}{carriers(c)}" if k else f"{c} empty" for c, k in count.items())) if paths is f["advanced"]: q = [c for c in QUALITY if count.get(c)] print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, " "tools/DATA.md" if q else "empty")) days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window for p in others.pop("metrics", []): m = METRIC_RE.match(os.path.basename(p)) if not m: continue day = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc) if (start is None or day.date() >= start.date()) and (end is None or day.date() <= end.date()): name = m.group(1) days[name.split("_", 1)[1] if "_" in name else name] += 1 if days: print(f" System metrics (metrics/, one CSV per metric and per day; read by metrics.py): {len(days)}: " + ", ".join(f"{k} ({n} d)" for k, n in sorted(days.items()))) for title, paths in (("Camera images", f["images"]), ("Differential images (images_diff/; no tool reads them)", glob.glob(os.path.join(folder, "images_diff", "*")))): times = [h for h in map(image_time, paths) if h and filter_window([{"t": h}], start, end)] if paths: print(f" {title}: {len(times)}") if others: print(" Others: " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(others.items()))) def images(paths, start=None, end=None): times = sorted(h for h in map(image_time, paths) if h) print(f"Images: {len(paths)}" + (f", from {iso(times[0])} to {iso(times[-1])}" if times else "")) unknown = [p for p in paths if image_time(p) is None] if unknown: print(f" names without a recognised time: {len(unknown)}") times = [h for h in times if filter_window([{"t": h}], start, end)] if len(times) < 2: return # Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot): # one line per change of gap, without losing any of the times covered. pairs = [(a, b, round((b - a).total_seconds() / 300) * 5) for a, b in zip(times, times[1:])] what = "over the window" if start or end else "between the first and the last" print(f" gap between two images {what}, per run of equal gaps (within 5 min):") for g in group(pairs, lambda p, q: p[2] == q[2]): n, e = len(g), g[0][2] print(f" {iso(g[0][0])} → {iso(g[-1][1])}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min")) def missing(instants, cadence): """Gaps between two present instants: [(before, after, number of absent steps)].""" return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0] def measurements(by, targets=None): if not by: print('\nNo measurement: no "advanced" export in this folder.') return instants = sorted({m["t"] for ms in by.values() for m in ms}) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants") print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min") gaps = missing(instants, cadence) print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}") for a, b, n in gaps: print(f" between {iso(a)} and {iso(b)}: {n}") names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) refs = [s for s in names if is_reference(by[s])] print(f"\nSensors: {len(by)} ({len(refs)} references, {len(by) - len(refs)} targets)") empty = [s for s in names if not any(m["pattern_type"] for m in by[s])] if empty: print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled " "(references only, tools/DATA.md)") targets = targets or {} if targets: print(" distance (m) and mm/px: targets.csv (data.load_targets); \"-\": not in it, or not filled in " "(distance 1000 mm); \"?\": its lines (n) disagree, no corr_x_mm / corr_y_mm") else: print(" no targets.csv: no distance, no corr_x_mm / corr_y_mm") print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}" f" {'distance':>9} {'mm/px':>8}") for s in names: ms = by[s] threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms})) t = targets.get(s, {}) mark = f"?({t['lines']})" if t.get("lines", 0) > 1 else "-" distance = mark if t.get("distance_mm") is None else f"{t['distance_mm'] / 1000:g}" factor = mark if t.get("mm_per_px") is None else f"{t['mm_per_px']:.3f}" print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} " f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8} {distance:>9} {factor:>8}") def camera(folder, by, start=None, end=None): """Camera poses, and the measurement instants without a pose (the camera was not registered).""" poses = filter_window(load_camera(folder), start, end) instants = sorted({m["t"] for ms in by.values() for m in filter_window(ms, start, end)}) what = "in the window" if start or end else "over the whole period" if not poses: print(f"\nCamera pose {what}: none") return print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}") runs = without_pose(instants, poses) print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}") if start or end: for a, b, n in runs: print(f" {span(a, b, n)}") def window(by, start, end): """Absent steps and rejections dated over the start/end window.""" instants = sorted({m["t"] for ms in by.values() for m in ms}) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) series = {s: {m["t"]: m for m in filter_window(ms, start, end)} for s, ms in by.items()} inside = sorted({t for d in series.values() for t in d}) print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present") names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):") gaps = missing(inside, cadence) for a, b, n in gaps: print(f" between {iso(a)} and {iso(b)}: {n}") if not gaps: print(" (none)") print("Present instants where sensors have no measurement:") partial = [(t, [s for s in names if t not in series[s]]) for t in inside] for t, lacking in partial: if lacking: print(f" {iso(t)}: {len(lacking)} sensors: {','.join(lacking)}") if not any(m for _, m in partial): print(" (none)") print("Rejected measurements in the window, per sensor (rejected of measurements; start → end of each run of " "consecutive steps, count), then the share rejected per UTC day (MM-DD:%):") for s in names: rej = [t for t in sorted(series[s]) if rejected(series[s][t])] if rej: n = len(series[s]) print(f" {s} ({len(rej)} of {n}, {100 * len(rej) / n:.1f} %): " + "; ".join(span(g[0], g[-1], len(g)) for g in group(rej, lambda a, b: b - a <= cadence * 1.5))) days = collections.defaultdict(lambda: [0, 0]) for t, m in series[s].items(): days[t.date()][0] += 1 days[t.date()][1] += rejected(m) print(f" {s} per day, {sum(1 for v in days.values() if v[1])} of {len(days)} days with a rejection: " + " ".join(f"{d:%m-%d}:{100 * r / k:.0f}" for d, (k, r) in sorted(days.items()))) if not any(rejected(m) for d in series.values() for m in d.values()): print(" (none)") refs = [s for s in names if is_reference(by[s])] # inlier says which references the compensation retains; some exports do not fill it (and sometimes # only reproj_err): filled reproj_err is taken then, and the title says so. col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None) if col is None: print("Retained references by the compensation: columns inlier and reproj_err absent or empty.") return retained = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None) def mark(s, t): m = series[s].get(t) return "absent" if m is None else "r" if rejected(m) else "empty" if m[col] is None else f"{m[col]:g}" out = [(t, tuple((s, mark(s, t)) for s in refs if t not in series[s] or not retained(series[s][t]))) for t in inside] count = collections.Counter(len(refs) - len(h) for _, h in out) what = "inlier = 1" if col == "inlier" else "inlier empty over the window: reproj_err filled instead" print(f"Retained references by the compensation ({what}) out of {len(refs)}, number of instants: " + ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True))) print("Instants where references are not retained (runs of consecutive steps with the same references out; " f"r = rejected, empty = {col} empty, otherwise the inlier value):") runs = group([(i, t, h) for i, (t, h) in enumerate(out) if h], lambda a, b: b[0] == a[0] + 1 and b[2] == a[2]) for g in runs: h = g[0][2] print(f" {span(g[0][1], g[-1][1], len(g))}: {len(refs) - len(h)} retained; out: " + ", ".join(f"{s} ({q})" for s, q in h)) if not runs: print(" (none)") def main(folder, start=None, end=None, full=False): print(f"Folder: {folder}") f = find_files(folder) files(f, full) contents(folder, f, start, end) images(f["images"], start, end) by = load(folder) measurements(by, load_targets(folder)) if by: camera(folder, by) if by and (start or end): window(by, start, end) camera(folder, by, start, end) if __name__ == "__main__": a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folder") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) a.add_argument("--files", action="store_true", help="full list of the files (otherwise: the first and the last beyond 5)") o = a.parse_args() main(o.folder, o.start, o.end, o.files)