2026-11-13 12:40:30 +01:00
|
|
|
|
"""Detector of the No data family (the families document): moments without a measurement (the cadence of the "advanced"
|
2026-11-13 12:19:52 +01:00
|
|
|
|
export is broken).
|
|
|
|
|
|
|
|
|
|
|
|
Cadence: median gap between two measurement instants, all sensors together. A hole: more than MISSING missing steps
|
|
|
|
|
|
between two measurements (gap > MISSING + 1 cadences); the episode runs from the last measurement to the next one.
|
|
|
|
|
|
End of the window read (end, when the caller gives it: the chunked run of categories.py): the same rule between the last
|
|
|
|
|
|
measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of
|
|
|
|
|
|
the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the
|
|
|
|
|
|
export has nothing.
|
2026-11-13 12:45:14 +01:00
|
|
|
|
Usage: python3 scripts/no_data.py <station folder> (prints the episodes)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
"""
|
|
|
|
|
|
import os
|
|
|
|
|
|
import statistics
|
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
|
|
|
|
|
|
from data import iso, load # noqa: E402
|
|
|
|
|
|
|
|
|
|
|
|
MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def duration(s):
|
|
|
|
|
|
h, m = divmod(round(s / 60), 60)
|
|
|
|
|
|
return f"{h} h {m:02d} min" if h else f"{m} min"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def detect(measurements, end=None):
|
|
|
|
|
|
"""end: end of the window read (datetime, excluded), optional; without it, a final hole is not seen."""
|
|
|
|
|
|
ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms})
|
|
|
|
|
|
if len(ts) < 3:
|
|
|
|
|
|
return []
|
|
|
|
|
|
gaps = [(b - a).total_seconds() for a, b in zip(ts, ts[1:])]
|
|
|
|
|
|
step = statistics.median(gaps)
|
|
|
|
|
|
out = [(a, b, g, "") for a, b, g in zip(ts, ts[1:], gaps)]
|
|
|
|
|
|
if end: # end excluded: the last expected measurement is at end − 1 step, the gap counts as between two measurements
|
|
|
|
|
|
out.append((ts[-1], end, (end - ts[-1]).total_seconds(), ", up to the end of the period"))
|
|
|
|
|
|
return [{"start": iso(a), "end": iso(b), "kind": "No data", "label": "No data",
|
|
|
|
|
|
"text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).",
|
|
|
|
|
|
"ignore": False, "sensors": []}
|
|
|
|
|
|
for a, b, g, tail in out if g > (MISSING + 1) * step]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
|
for e in detect(load(sys.argv[1])):
|
|
|
|
|
|
print(e)
|