no_data: the cadence of common.py, shorter

Benchmark unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 16:39:11 +01:00
1 parent ec4b900fa0
commit 3a22dae7ce
1 file changed
+10 -32
+10 -32
View File
@@ -1,42 +1,20 @@
"""Detector of the No data family (the families document): moments without a measurement (the cadence of the "advanced"
export is broken).
"""No data family: no measurement in the export for more than MISSING steps of its cadence (common.cadence), between two
measurements or, when the caller gives the end of the window read, between the last measurement and that end."""
from common import cadence, episode
Cadence: median gap between two measurement instants, all sensors together; at least an hour divided by the median
number of instants of the clock hours that have one: a station measuring in bursts (three measurements a minute apart
every 15 min) has a median gap of a minute, but a cycle of 5 min; at the median gap, every gap between two bursts was a
hole, a whole day of No data (wf-improve, batch 2026-11-05, S1). A hole: more than MISSING missing steps between two
measurements (gap > MISSING + 1 cadences); the episode runs from the last measurement to the next one.
End of the window read (end, when the caller gives it: the chunked run of categories.py): the same rule between the last
measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of
the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the
export has nothing.
"""
import statistics
from data import iso
MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23)
MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23); also up to the end (CLARIDEN, batch 01)
def duration(s):
h, m = divmod(round(s / 60), 60)
def duration(d):
h, m = divmod(round(d.total_seconds() / 60), 60)
return f"{h} h {m:02d} min" if h else f"{m} min"
def detect(measurements, end=None):
"""end: end of the window read (datetime, excluded), optional; without it, a final hole is not seen."""
ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms})
if len(ts) < 3:
return []
gaps = [(b - a).total_seconds() for a, b in zip(ts, ts[1:])]
per_hour = {}
for t in ts:
per_hour[t.replace(minute=0)] = per_hour.get(t.replace(minute=0), 0) + 1
step = max(statistics.median(gaps), 3600 / statistics.median(per_hour.values())) # bursts: the cycle
out = [(a, b, g, "") for a, b, g in zip(ts, ts[1:], gaps)]
if end: # end excluded: the last expected measurement is at end − 1 step, the gap counts as between two measurements
out.append((ts[-1], end, (end - ts[-1]).total_seconds(), ", up to the end of the period"))
return [{"start": iso(a), "end": iso(b), "kind": "No data", "label": "No data",
"text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).",
"ignore": False, "sensors": []}
for a, b, g, tail in out if g > (MISSING + 1) * step]
step = cadence(measurements)
holes = [(a, b, "") for a, b in zip(ts, ts[1:])] + ([(ts[-1], end, ", up to the end of the period")] if end else [])
return [episode(a, b, "No data", "No data", f"No measurement in the export for {duration(b - a)}{tail} (usually one "
f"every {duration(step)}).", [], False) for a, b, tail in holes if b - a > (MISSING + 1) * step]