auto: labels requalified under the 5-noise rules (only what stands out from the noise): 32 analyses refined, the 6 paused periods verified, the long labels requalified with the fixes the period analysts found; 1153 -> 961 labels

- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them)
- SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules
- long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed
- reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 23:58:00 +01:00
1 parent c779de7720
commit ddb17c2189
321 files changed
+18464 -3426

No files matched your search

@@ -0,0 +1,18 @@
"""Summarise ranges.py output: per sensor and column, measurements beyond k noises within [start,end), max |noises|, max |value|."""
import re, sys
f, start, end = sys.argv[1], sys.argv[2], sys.argv[3]
rx = re.compile(r'^(\S+?)\s*(2026-\d\d-\d\dT\d\d:\d\dZ)\s+(2026-\S+)\s+(\d+)\s+([+-][\d.e+-]+)\s+([+-]?[\d.]+)\s+(\S+)\s+(\d+)')
col = None; st = {}
for line in open(f):
m = re.match(r'^(corr_\w+): ranges', line)
if m: col = m.group(1); continue
m = rx.match(line)
if not m or col is None: continue
s, a, b, n, v, z = m.group(1), m.group(2), m.group(3), int(m.group(4)), float(m.group(5)), float(m.group(6))
if b < start or a >= end: continue
d = st.setdefault((s, col), [0, 0, 0.0, 0.0, 0])
d[0] += n; d[1] += 1
if abs(z) > abs(d[2]): d[2] = z; d[3] = v
d[4] = max(d[4], n)
for (s, c), d in sorted(st.items()):
print(f"{s:22s} {c:10s} meas>{5}: {d[0]:4d} in {d[1]:3d} ranges, longest {d[4]:3d}, extreme {d[2]:+6.1f} noises ({d[3]:+.3f})")
@@ -0,0 +1,22 @@
"""Per sensor and column, over a window: accepted measurements by hour of day, median deviation from the night
level (median of 20-03 UTC of the whole window), in own noises; and count of accepted points beyond K noises
from the night level. Usage: daynight.py START END SENSORS [K]"""
import sys, statistics
sys.path.insert(0, "tools/scripts")
import data, common
start, end = data.parse_time(sys.argv[1]), data.parse_time(sys.argv[2])
K = float(sys.argv[4]) if len(sys.argv) > 4 else 5
by = data.load("data/NIEDERHORN")
for s in sys.argv[3].split(","):
ms = [m for m in data.filter_window(by[s], start, end)]
for col in ("corr_x", "corr_y"):
v = [(m["t"], m[col]) for m in ms if m[col] is not None]
if len(v) < 10: continue
noise = common.own_noise([x for _, x in v])
night = [x for t, x in v if t.hour >= 20 or t.hour <= 3]
lvl = statistics.median(night) if night else statistics.median([x for _, x in v])
hours = {}
for t, x in v: hours.setdefault(t.hour, []).append((x - lvl) / noise)
prof = " ".join(f"{h:02d}:{statistics.median(hours[h]):+5.1f}({len(hours[h])})" for h in sorted(hours))
beyond = sum(1 for _, x in v if abs(x - lvl) / noise > K)
print(f"{s:20s} {col} n={len(v)} noise={noise:.4f} beyond{K:g}={beyond}\n {prof}")
@@ -0,0 +1,46 @@
"""For each studied target, over a window: corr minus the median of the healthy targets at the same instant
(common mode removed), then its own noise (median abs consecutive diff of the raw corr, as ranges.py) and the
measurements whose residual deviates from its rolling-1d median by more than K noises. Also the healthy targets,
leave-one-out, for comparison. Usage: specific.py START END SENSOR[,SENSOR] [K]"""
import sys, statistics
from datetime import timedelta
sys.path.insert(0, "tools/scripts")
import data, common
start, end = data.parse_time(sys.argv[1]), data.parse_time(sys.argv[2])
studied = sys.argv[3].split(",")
K = float(sys.argv[4]) if len(sys.argv) > 4 else 5
by = data.load("data/NIEDERHORN")
healthy = ["PWD_RMP_B12_020_C09","PWD_IMP_B08_025_C09","PWD_RMP_B15_018_C09","PWD_IMP_B07_017_C09","PWD_RMP_B12_018_C09",
"PWD_RMP_B12_017_C09","PWD_RMP_B15_019_C09","PWD_RMP_B15_020_C09","PWD_RMP_B17_026_C09",
"e02","e06","e09","e10","e13","e14","e15"]
win = {s: [m for m in data.filter_window(ms, start, end)] for s, ms in by.items()}
def series(s, col):
return {m["t"]: m[col] for m in win[s] if m[col] is not None}
for col in ("corr_x", "corr_y"):
H = {s: series(s, col) for s in healthy}
print(f"== {col} (K={K}, rolling ±1d, common mode = median of healthy targets)")
for s in studied + healthy:
own = series(s, col)
if len(own) < 10:
print(f"{s:22s} n={len(own)} (no curve)"); continue
noise = common.own_noise([own[t] for t in sorted(own)])
res = {}
for t, v in own.items():
others = [H[h][t] for h in healthy if h != s and t in H[h]]
if len(others) >= 5:
res[t] = v - statistics.median(others)
ts = sorted(res)
out = []
for t in ts:
nb = [res[u] for u in ts if u != t and abs((u - t).total_seconds()) <= 86400]
if len(nb) < 5: continue
d = (res[t] - statistics.median(nb)) / noise
if abs(d) > K: out.append((t, d))
ext = max(out, key=lambda x: abs(x[1])) if out else None
kmm = None
mm = [m for m in win[s] if m[col + "_mm"] is not None and m[col] not in (None, 0)]
if mm: kmm = mm[0][col + "_mm"] / mm[0][col]
days = len({t.date() for t, _ in out})
e = f"extreme {ext[1]:+.1f} noises ({ext[1]*noise:+.3f} px" + (f", {ext[1]*noise*kmm:+.2f} mm" if kmm else "") + f") at {data.iso(ext[0])}" if ext else ""
tag = "*" if s in studied else " "
print(f"{tag}{s:22s} n={len(own):4d} noise={noise:.4f} beyond: {len(out):3d} ({100*len(out)/max(1,len(res)):4.1f} %) on {days:2d} days {e}")