DATA.md: what the data are, from labeling/knowledge (formats, columns, quality metrics, images)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f28bd19e04
commit
4bfbad4db8
7 files changed
+73
-12
No files matched your search
+2
-2
@@ -1,6 +1,6 @@
|
||||
"""Reading a station's data for the diagnostic agent (standard library).
|
||||
|
||||
Formats: labeling/knowledge/displacement-columns.md. Every time is UTC.
|
||||
Formats: tools/DATA.md. Every time is UTC.
|
||||
"""
|
||||
import csv
|
||||
import glob
|
||||
@@ -126,7 +126,7 @@ def load_metrics(folder):
|
||||
def is_reference(measurements):
|
||||
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
|
||||
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
|
||||
none, filled reproj_err or inlier (references only: labeling/knowledge/displacement-columns.md)."""
|
||||
none, filled reproj_err or inlier (references only: tools/DATA.md)."""
|
||||
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
|
||||
if t is not None:
|
||||
return t == "global_ref"
|
||||
|
||||
@@ -76,7 +76,7 @@ def files(f, full=False):
|
||||
list_paths(f["advanced"], full)
|
||||
duplicates(f["advanced"])
|
||||
print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, "
|
||||
"local time without a zone; see labeling/knowledge/displacement-columns.md)")
|
||||
"local time without a zone; see tools/DATA.md)")
|
||||
list_paths(f["hourly"], full)
|
||||
print(f"Camera movements (pose at each registration): {len(f['camera'])}")
|
||||
list_paths(f["camera"], full)
|
||||
@@ -125,7 +125,7 @@ def contents(folder, f, start=None, end=None):
|
||||
if paths is f["advanced"]:
|
||||
q = [c for c in QUALITY if count.get(c)]
|
||||
print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, "
|
||||
"labeling/knowledge/quality-metrics.md" if q else "empty"))
|
||||
"tools/DATA.md" if q else "empty"))
|
||||
days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window
|
||||
for p in others.pop("metrics", []):
|
||||
m = METRIC_RE.match(os.path.basename(p))
|
||||
@@ -190,7 +190,7 @@ def measurements(by):
|
||||
empty = [s for s in names if not any(m["pattern_type"] for m in by[s])]
|
||||
if empty:
|
||||
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
|
||||
"(references only, labeling/knowledge/displacement-columns.md)")
|
||||
"(references only, tools/DATA.md)")
|
||||
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
|
||||
for s in names:
|
||||
ms = by[s]
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ Usage: python3 tools/scripts/quality.py <folder> [--start T] [--end T] [--sensor
|
||||
[--metrics flux,snr,dominance] [--labels F] [--zone T1/T2 ...] [--out D]
|
||||
python3 tools/scripts/quality.py --self-test (check of the ratio computation)
|
||||
|
||||
Definitions: labeling/knowledge/quality-metrics.md. Columns filled on recent cameras only; on a folder where
|
||||
Definitions: tools/DATA.md. Columns filled on recent cameras only; on a folder where
|
||||
they are empty, the tool says so and stops.
|
||||
Reference of a sensor: the median of each metric over its measurements in the --calm windows, at the same UTC hour of the day
|
||||
(24 slots; a slot without a calm measurement takes the median of all calm hours). Without --calm: the whole folder.
|
||||
@@ -374,7 +374,7 @@ def main():
|
||||
sys.exit(f"No \"advanced\" export in {a.folder} (see inventory.py).")
|
||||
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
|
||||
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
|
||||
"labeling/knowledge/quality-metrics.md).")
|
||||
"tools/DATA.md).")
|
||||
names = a.sensors.split(",") if a.sensors else sorted(every, key=lambda s: (not data.is_reference(every[s]), s))
|
||||
missing = [s for s in names if s not in every]
|
||||
if missing:
|
||||
|
||||
Reference in new issue
Block a user