DATA.md: what the data are, from labeling/knowledge (formats, columns, quality metrics, images)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 12:41:44 +01:00
1 parent f28bd19e04
commit 4bfbad4db8
7 files changed
+73 -12

No files matched your search

+2 -2
View File
@@ -1,6 +1,6 @@
"""Reading a station's data for the diagnostic agent (standard library).
Formats: labeling/knowledge/displacement-columns.md. Every time is UTC.
Formats: tools/DATA.md. Every time is UTC.
"""
import csv
import glob
@@ -126,7 +126,7 @@ def load_metrics(folder):
def is_reference(measurements):
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
none, filled reproj_err or inlier (references only: labeling/knowledge/displacement-columns.md)."""
none, filled reproj_err or inlier (references only: tools/DATA.md)."""
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
+3 -3
View File
@@ -76,7 +76,7 @@ def files(f, full=False):
list_paths(f["advanced"], full)
duplicates(f["advanced"])
print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, "
"local time without a zone; see labeling/knowledge/displacement-columns.md)")
"local time without a zone; see tools/DATA.md)")
list_paths(f["hourly"], full)
print(f"Camera movements (pose at each registration): {len(f['camera'])}")
list_paths(f["camera"], full)
@@ -125,7 +125,7 @@ def contents(folder, f, start=None, end=None):
if paths is f["advanced"]:
q = [c for c in QUALITY if count.get(c)]
print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, "
"labeling/knowledge/quality-metrics.md" if q else "empty"))
"tools/DATA.md" if q else "empty"))
days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window
for p in others.pop("metrics", []):
m = METRIC_RE.match(os.path.basename(p))
@@ -190,7 +190,7 @@ def measurements(by):
empty = [s for s in names if not any(m["pattern_type"] for m in by[s])]
if empty:
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
"(references only, labeling/knowledge/displacement-columns.md)")
"(references only, tools/DATA.md)")
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
for s in names:
ms = by[s]
+2 -2
View File
@@ -8,7 +8,7 @@ Usage: python3 tools/scripts/quality.py <folder> [--start T] [--end T] [--sensor
[--metrics flux,snr,dominance] [--labels F] [--zone T1/T2 ...] [--out D]
python3 tools/scripts/quality.py --self-test (check of the ratio computation)
Definitions: labeling/knowledge/quality-metrics.md. Columns filled on recent cameras only; on a folder where
Definitions: tools/DATA.md. Columns filled on recent cameras only; on a folder where
they are empty, the tool says so and stops.
Reference of a sensor: the median of each metric over its measurements in the --calm windows, at the same UTC hour of the day
(24 slots; a slot without a calm measurement takes the median of all calm hours). Without --calm: the whole folder.
@@ -374,7 +374,7 @@ def main():
sys.exit(f"No \"advanced\" export in {a.folder} (see inventory.py).")
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
"labeling/knowledge/quality-metrics.md).")
"tools/DATA.md).")
names = a.sensors.split(",") if a.sensors else sorted(every, key=lambda s: (not data.is_reference(every[s]), s))
missing = [s for s in names if s not in every]
if missing: