diff --git a/scripts/no_data.py b/scripts/no_data.py index 660681d..7b2b64f 100644 --- a/scripts/no_data.py +++ b/scripts/no_data.py @@ -1,42 +1,20 @@ -"""Detector of the No data family (the families document): moments without a measurement (the cadence of the "advanced" -export is broken). +"""No data family: no measurement in the export for more than MISSING steps of its cadence (common.cadence), between two +measurements or, when the caller gives the end of the window read, between the last measurement and that end.""" +from common import cadence, episode -Cadence: median gap between two measurement instants, all sensors together; at least an hour divided by the median -number of instants of the clock hours that have one: a station measuring in bursts (three measurements a minute apart -every 15 min) has a median gap of a minute, but a cycle of 5 min; at the median gap, every gap between two bursts was a -hole, a whole day of No data (wf-improve, batch 2026-11-05, S1). A hole: more than MISSING missing steps between two -measurements (gap > MISSING + 1 cadences); the episode runs from the last measurement to the next one. -End of the window read (end, when the caller gives it: the chunked run of categories.py): the same rule between the last -measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of -the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the -export has nothing. -""" -import statistics - -from data import iso - -MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23) +MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23); also up to the end (CLARIDEN, batch 01) -def duration(s): - h, m = divmod(round(s / 60), 60) +def duration(d): + h, m = divmod(round(d.total_seconds() / 60), 60) return f"{h} h {m:02d} min" if h else f"{m} min" def detect(measurements, end=None): - """end: end of the window read (datetime, excluded), optional; without it, a final hole is not seen.""" ts = sorted({m["t"].replace(second=0, microsecond=0) for ms in measurements.values() for m in ms}) if len(ts) < 3: return [] - gaps = [(b - a).total_seconds() for a, b in zip(ts, ts[1:])] - per_hour = {} - for t in ts: - per_hour[t.replace(minute=0)] = per_hour.get(t.replace(minute=0), 0) + 1 - step = max(statistics.median(gaps), 3600 / statistics.median(per_hour.values())) # bursts: the cycle - out = [(a, b, g, "") for a, b, g in zip(ts, ts[1:], gaps)] - if end: # end excluded: the last expected measurement is at end − 1 step, the gap counts as between two measurements - out.append((ts[-1], end, (end - ts[-1]).total_seconds(), ", up to the end of the period")) - return [{"start": iso(a), "end": iso(b), "kind": "No data", "label": "No data", - "text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).", - "ignore": False, "sensors": []} - for a, b, g, tail in out if g > (MISSING + 1) * step] + step = cadence(measurements) + holes = [(a, b, "") for a, b in zip(ts, ts[1:])] + ([(ts[-1], end, ", up to the end of the period")] if end else []) + return [episode(a, b, "No data", "No data", f"No measurement in the export for {duration(b - a)}{tail} (usually one " + f"every {duration(step)}).", [], False) for a, b, tail in holes if b - a > (MISSING + 1) * step]