auto: labels requalified under the 5-noise rules (only what stands out from the noise): 32 analyses refined, the 6 paused periods verified, the long labels requalified with the fixes the period analysts found; 1153 -> 961 labels
- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them) - SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules - long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed - reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert] Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c779de7720
commit
ddb17c2189
321 files changed
+18464
-3426
No files matched your search
@@ -0,0 +1,18 @@
|
||||
"""Summarise ranges.py output: per sensor and column, measurements beyond k noises within [start,end), max |noises|, max |value|."""
|
||||
import re, sys
|
||||
f, start, end = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
rx = re.compile(r'^(\S+?)\s*(2026-\d\d-\d\dT\d\d:\d\dZ)\s+(2026-\S+)\s+(\d+)\s+([+-][\d.e+-]+)\s+([+-]?[\d.]+)\s+(\S+)\s+(\d+)')
|
||||
col = None; st = {}
|
||||
for line in open(f):
|
||||
m = re.match(r'^(corr_\w+): ranges', line)
|
||||
if m: col = m.group(1); continue
|
||||
m = rx.match(line)
|
||||
if not m or col is None: continue
|
||||
s, a, b, n, v, z = m.group(1), m.group(2), m.group(3), int(m.group(4)), float(m.group(5)), float(m.group(6))
|
||||
if b < start or a >= end: continue
|
||||
d = st.setdefault((s, col), [0, 0, 0.0, 0.0, 0])
|
||||
d[0] += n; d[1] += 1
|
||||
if abs(z) > abs(d[2]): d[2] = z; d[3] = v
|
||||
d[4] = max(d[4], n)
|
||||
for (s, c), d in sorted(st.items()):
|
||||
print(f"{s:22s} {c:10s} meas>{5}: {d[0]:4d} in {d[1]:3d} ranges, longest {d[4]:3d}, extreme {d[2]:+6.1f} noises ({d[3]:+.3f})")
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Per sensor and column, over a window: accepted measurements by hour of day, median deviation from the night
|
||||
level (median of 20-03 UTC of the whole window), in own noises; and count of accepted points beyond K noises
|
||||
from the night level. Usage: daynight.py START END SENSORS [K]"""
|
||||
import sys, statistics
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data, common
|
||||
start, end = data.parse_time(sys.argv[1]), data.parse_time(sys.argv[2])
|
||||
K = float(sys.argv[4]) if len(sys.argv) > 4 else 5
|
||||
by = data.load("data/NIEDERHORN")
|
||||
for s in sys.argv[3].split(","):
|
||||
ms = [m for m in data.filter_window(by[s], start, end)]
|
||||
for col in ("corr_x", "corr_y"):
|
||||
v = [(m["t"], m[col]) for m in ms if m[col] is not None]
|
||||
if len(v) < 10: continue
|
||||
noise = common.own_noise([x for _, x in v])
|
||||
night = [x for t, x in v if t.hour >= 20 or t.hour <= 3]
|
||||
lvl = statistics.median(night) if night else statistics.median([x for _, x in v])
|
||||
hours = {}
|
||||
for t, x in v: hours.setdefault(t.hour, []).append((x - lvl) / noise)
|
||||
prof = " ".join(f"{h:02d}:{statistics.median(hours[h]):+5.1f}({len(hours[h])})" for h in sorted(hours))
|
||||
beyond = sum(1 for _, x in v if abs(x - lvl) / noise > K)
|
||||
print(f"{s:20s} {col} n={len(v)} noise={noise:.4f} beyond{K:g}={beyond}\n {prof}")
|
||||
@@ -0,0 +1,46 @@
|
||||
"""For each studied target, over a window: corr minus the median of the healthy targets at the same instant
|
||||
(common mode removed), then its own noise (median abs consecutive diff of the raw corr, as ranges.py) and the
|
||||
measurements whose residual deviates from its rolling-1d median by more than K noises. Also the healthy targets,
|
||||
leave-one-out, for comparison. Usage: specific.py START END SENSOR[,SENSOR] [K]"""
|
||||
import sys, statistics
|
||||
from datetime import timedelta
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data, common
|
||||
start, end = data.parse_time(sys.argv[1]), data.parse_time(sys.argv[2])
|
||||
studied = sys.argv[3].split(",")
|
||||
K = float(sys.argv[4]) if len(sys.argv) > 4 else 5
|
||||
by = data.load("data/NIEDERHORN")
|
||||
healthy = ["PWD_RMP_B12_020_C09","PWD_IMP_B08_025_C09","PWD_RMP_B15_018_C09","PWD_IMP_B07_017_C09","PWD_RMP_B12_018_C09",
|
||||
"PWD_RMP_B12_017_C09","PWD_RMP_B15_019_C09","PWD_RMP_B15_020_C09","PWD_RMP_B17_026_C09",
|
||||
"e02","e06","e09","e10","e13","e14","e15"]
|
||||
win = {s: [m for m in data.filter_window(ms, start, end)] for s, ms in by.items()}
|
||||
def series(s, col):
|
||||
return {m["t"]: m[col] for m in win[s] if m[col] is not None}
|
||||
for col in ("corr_x", "corr_y"):
|
||||
H = {s: series(s, col) for s in healthy}
|
||||
print(f"== {col} (K={K}, rolling ±1d, common mode = median of healthy targets)")
|
||||
for s in studied + healthy:
|
||||
own = series(s, col)
|
||||
if len(own) < 10:
|
||||
print(f"{s:22s} n={len(own)} (no curve)"); continue
|
||||
noise = common.own_noise([own[t] for t in sorted(own)])
|
||||
res = {}
|
||||
for t, v in own.items():
|
||||
others = [H[h][t] for h in healthy if h != s and t in H[h]]
|
||||
if len(others) >= 5:
|
||||
res[t] = v - statistics.median(others)
|
||||
ts = sorted(res)
|
||||
out = []
|
||||
for t in ts:
|
||||
nb = [res[u] for u in ts if u != t and abs((u - t).total_seconds()) <= 86400]
|
||||
if len(nb) < 5: continue
|
||||
d = (res[t] - statistics.median(nb)) / noise
|
||||
if abs(d) > K: out.append((t, d))
|
||||
ext = max(out, key=lambda x: abs(x[1])) if out else None
|
||||
kmm = None
|
||||
mm = [m for m in win[s] if m[col + "_mm"] is not None and m[col] not in (None, 0)]
|
||||
if mm: kmm = mm[0][col + "_mm"] / mm[0][col]
|
||||
days = len({t.date() for t, _ in out})
|
||||
e = f"extreme {ext[1]:+.1f} noises ({ext[1]*noise:+.3f} px" + (f", {ext[1]*noise*kmm:+.2f} mm" if kmm else "") + f") at {data.iso(ext[0])}" if ext else ""
|
||||
tag = "*" if s in studied else " "
|
||||
print(f"{tag}{s:22s} n={len(own):4d} noise={noise:.4f} beyond: {len(out):3d} ({100*len(out)/max(1,len(res)):4.1f} %) on {days:2d} days {e}")
|
||||
Reference in new issue
Block a user