auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 15:13:44 +01:00
1 parent df979301b5
commit 2a5d87659b
279 files changed
+10758 -710

No files matched your search

@@ -0,0 +1,77 @@
"""Build labels.jsonl: existing labels kept as they are (without id), three corrected, seven added."""
import json
D = 'labeling/auto/analyses/NIEDERHORN_2026-10-21T1200_2026-11-01T0000/'
PLANT = ("It is the foreground plant, shaken by the wind, which, according to the expert, passes in front of this target. "
"The structure does not move: ignore this target's curve over the range.")
corrected = {
'NIEDERHORN-105': dict(start='2026-10-23T09:00Z', end='2026-10-23T16:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone: at 10:00 its corr_x jumps up and its corr_y down strongly for one measurement; at 12:00 and 13:00 "
"its recognition error jumps almost to the maximum and the measurement is rejected; at 14:00 its corr_x jumps up strongly "
"again and its corr_y down; at 15:00 its corr_x is still a little off, back at 16:00; its neighbours do not move. Cause seen: "
"on the 10:00 thumbnail a leaf touches the spot; on the 12:00 and 13:00 thumbnails, at daybreak, blurred leaves cover the "
"spot, almost invisible. " + PLANT),
'NIEDERHORN-114': dict(start='2026-10-26T01:00Z', end='2026-10-26T05:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 02:00 its corr_y jumps up clearly, at 04:00 its corr_x jumps up and its "
"corr_y down strongly, each time for one measurement, back in between and after; its recognition error rises at both; its "
"neighbours do not move. Cause seen on the 04:00 thumbnail: the tips of leaves touch the spot; on the 02:00 thumbnail, leaves "
"are just next to it. " + PLANT),
'NIEDERHORN-119': dict(start='2026-10-27T06:00Z', end='2026-10-27T10:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone: at 07:00 its corr_y rises clearly for one measurement, at 08:00 its recognition error jumps above "
"the threshold and the measurement is rejected, at 09:00 its corr_x jumps up and its corr_y down strongly; back at 10:00; its "
"neighbours do not move. Cause seen on the 08:00 thumbnail: leaves touch the spot; on the 07:00 thumbnail they are just next "
"to it. " + PLANT),
}
added = [
dict(start='2026-10-22T20:00Z', end='2026-10-23T01:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 21:00 and again at 23:00 its corr_y jumps up clearly for one measurement "
"(its corr_x drops a little at 21:00), back at 22:00 in between; its recognition error rises at 20:00, 21:00 and 23:00; its "
"neighbours do not move. The 00:00 measurement is missing for all the sensors (labelled apart); back at 01:00. The 21:00 spike "
"falls in the thermal cycle of the day, labelled apart. Cause seen on the 20:00 and 21:00 thumbnails: blurred leaves sweep "
"across and next to the spot. " + PLANT),
dict(start='2026-10-23T04:00Z', end='2026-10-23T06:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 05:00 its corr_x jumps up and its corr_y drops clearly for one "
"measurement, then comes back; its neighbours do not move. On the 05:00 thumbnail, leaves are just next to the spot, at its "
"lower right. Cause deduced from the signature, the same as the other passages of the plant before this target on these days. "
+ PLANT),
dict(start='2026-10-25T16:00Z', end='2026-10-25T18:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 17:00 its corr_x jumps up clearly and its corr_y drops a little for one "
"measurement, then comes back; its recognition error rises at that measurement; its neighbours do not move. It falls in the "
"thermal cycle of the day, labelled apart. Cause seen on the 17:00 thumbnail: a blurred leaf sweeps just under the spot. "
+ PLANT),
dict(start='2026-10-26T06:00Z', end='2026-10-26T08:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 07:00 its corr_y jumps up clearly and its corr_x drops a little for one "
"measurement, then comes back; its recognition error rises at that measurement; its neighbours do not move. On the 07:00 "
"thumbnail, leaves are just next to the spot; the same plant made this target jump earlier in the night (labelled apart). "
+ PLANT),
dict(start='2026-10-26T12:00Z', end='2026-10-26T16:00Z', family='Thermal displacement', description=
"Thermal cycle of the day (26/10), short and weak. The housing temperature rises a little from 12:00 to 14:00, then stays "
"almost flat in the afternoon and drops slowly in the evening, much less than on the sunny days. The camera pose follows: "
"at 14:00 the tilt rises markedly and the rotation drops a little, the scale rises a little, the pan rises only slightly; "
"tilt and rotation are back at the next measurement, 16:00 (the 15:00 measurement is missing for all the sensors, labelled "
"apart), the pan by 17:00, the scale only slowly, during the night. Read against the overall movement, the references and "
"the targets move together in the image: it is the camera, not one reference. The compensation absorbs most of it: at "
"14:00 only the corr_x of PWD_IMP_B08_024_C09, PWD_IMP_B07_018_C09 and PWD_IMP_B08_025_C09 drops a little, as on the sunny "
"days, the corr_y hardly moves; the curves are back at 16:00. On the 14:00 image, the sun shines through thin clouds at the "
"top of the field. Deduced cause: the camera and its mount move with the housing temperature, warmed briefly by the sun; the "
"same signature as the daily cycles of the other days, weaker. The rejection of e06 and the rise of the recognition error of "
"PWD_REF10_C09 at that measurement are labelled apart (Visibility). Normal, nothing to do."),
dict(start='2026-10-26T16:00Z', end='2026-10-26T18:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 17:00 its corr_y jumps up and its corr_x drops a little for one "
"measurement, then comes back; its recognition error rises a little; its neighbours do not move. Cause seen on the 17:00 "
"thumbnail: a blurred leaf sweeps next to the spot. " + PLANT),
dict(start='2026-10-28T03:00Z', end='2026-10-28T05:00Z', family='Obstruction', description=
"PWD_IMP_B07_016_C09 alone, without any rejection: at 04:00 its corr_y jumps up clearly and its corr_x drops a little for one "
"measurement, then comes back; its neighbours do not move. Cause seen on the thumbnails: leaves appear right next to the spot "
"at 04:00, absent at 03:00 and 05:00. " + PLANT),
]
out = []
for l in open(D + 'work/existing.jsonl'):
d = json.loads(l)
i = d.pop('id')
out.append(corrected.get(i, d))
out += added
out.sort(key=lambda d: (d['start'], d['end']))
with open(D + 'labels.jsonl', 'w') as f:
for d in out:
f.write(json.dumps(d, ensure_ascii=False) + '\n')
print(len(out), 'labels;', sum('review' in d for d in out), 'with review')
@@ -0,0 +1,39 @@
"""Per instant: rejected sensors and corr jumps (vs rolling local median, in own noises), and the labels covering it.
usage: scan.py [labels_file] [K]"""
import json, sys, statistics as st
sys.path.insert(0, 'tools/scripts')
import data
from datetime import datetime, timezone, timedelta
D = 'labeling/auto/analyses/NIEDERHORN_2026-10-21T1200_2026-11-01T0000/'
ALWAYS = {'e03', 'e04', 'e05', 'e07', 'e08', 'e11', 'e12'}
lf = sys.argv[1] if len(sys.argv) > 1 else D + 'work/existing.jsonl'
K = float(sys.argv[2]) if len(sys.argv) > 2 else 5
labels = [json.loads(l) for f in (lf, D + 'work/spanning.jsonl') for l in open(f)]
for i, l in enumerate(labels):
l['s'] = data.parse_time(l['start'].replace('Z', '+00:00')); l['e'] = data.parse_time(l['end'].replace('Z', '+00:00'))
l.setdefault('id', f'new{i}')
lo = datetime(2026, 10, 19, tzinfo=timezone.utc); hi = datetime(2026, 11, 3, tzinfo=timezone.utc)
M = {s: [m for m in ms if lo <= m['t'] <= hi] for s, ms in data.load('data/NIEDERHORN').items()}
T = sorted({m['t'].replace(second=0, microsecond=0) for ms in M.values() for m in ms})
rej, dev = {}, {}
for s, ms in M.items():
for m in ms:
t = m['t'].replace(second=0, microsecond=0)
if data.rejected(m): rej.setdefault(t, []).append(s)
for c in ('corr_x', 'corr_y'):
v = [(m['t'].replace(second=0, microsecond=0), m[c]) for m in ms if m[c] is not None]
if len(v) < 20: continue
noise = st.median(abs(v[i + 1][1] - v[i][1]) for i in range(len(v) - 1)) or 1e-9
for i, (t, x) in enumerate(v):
nb = [y for _, y in v[max(0, i - 6):i] + v[i + 1:i + 7]]
d = (x - st.median(nb)) / noise
if abs(d) > K: dev.setdefault(t, {}).setdefault(s, []).append(f"{c[-1]}{d:+.0f}")
a, b = datetime(2026, 10, 21, 11, tzinfo=timezone.utc), datetime(2026, 11, 1, 1, tzinfo=timezone.utc)
sh = lambda x: x.replace('PWD_', '').replace('_C09', '')
for t in T:
if not a <= t <= b: continue
r = [sh(s) for s in rej.get(t, []) if s not in ALWAYS]
j = [f"{sh(s)}({'/'.join(v)})" for s, v in sorted(dev.get(t, {}).items())]
cov = [l['id'].replace('NIEDERHORN-', '') + l['family'][:3] for l in labels if l['s'] < t < l['e'] and l['family'] != 'Target issue']
if r or j:
print(t.strftime('%m-%dT%H'), 'REJ:', ','.join(sorted(r)), '| JMP:', ' '.join(j), '| cov:', ','.join(cov))
@@ -0,0 +1,9 @@
"""Print t, corr_x, corr_y, error_value of one sensor between two instants. usage: series.py SENSOR START END"""
import sys
sys.path.insert(0, 'tools/scripts')
import data
s, a, b = sys.argv[1], data.parse_time(sys.argv[2]), data.parse_time(sys.argv[3])
for m in data.load('data/NIEDERHORN')[s]:
if a <= m['t'] <= b:
f = lambda v: ' - ' if v is None else f'{v:+.3f}'
print(data.iso(m['t'])[5:16], f(m['corr_x']), f(m['corr_y']), f'{m["error_value"]:.3f}', 'REJ' if data.rejected(m) else '')
@@ -0,0 +1,22 @@
"""One-measurement spikes (both neighbours on the same side, the smaller step > K own noises) and isolated rejections,
at instants no label (except Target issue) covers. usage: spikes.py LABELS K"""
import json, sys, statistics as st
sys.path.insert(0, 'tools/scripts')
import data
D = 'labeling/auto/analyses/NIEDERHORN_2026-10-21T1200_2026-11-01T0000/'
labels = [json.loads(l) for f in (sys.argv[1], D + 'work/spanning.jsonl') for l in open(f)]
K = float(sys.argv[2])
for l in labels:
l['s'], l['e'] = data.parse_time(l['start'][:-1]), data.parse_time(l['end'][:-1])
a, b = data.parse_time('2026-10-21T12:00'), data.parse_time('2026-11-01T00:00')
cov = lambda t: [l.get('id', '?') for l in labels if l['s'] < t < l['e'] and l['family'] != 'Target issue']
for s, ms in data.load('data/NIEDERHORN').items():
for c in ('corr_x', 'corr_y'):
v = [(m['t'].replace(second=0, microsecond=0), m[c]) for m in ms if m[c] is not None]
if len(v) < 50: continue
noise = st.median(abs(v[i + 1][1] - v[i][1]) for i in range(len(v) - 1)) or 1e-9
for i in range(1, len(v) - 1):
t, x = v[i]
d1, d2 = x - v[i - 1][1], x - v[i + 1][1]
if d1 * d2 > 0 and min(abs(d1), abs(d2)) > K * noise and a < t < b and not cov(t):
print(data.iso(t), s, c, f'{min(abs(d1), abs(d2)) / noise:.1f}', 'sign', '+' if d1 > 0 else '-')
@@ -0,0 +1,30 @@
"""Per hour of given days: median over accepted targets of |corr step| from the previous measurement, in own noises,
and the number of targets whose step exceeds 4 noises. usage: steps.py HOURS(e.g. 9-16) DAY1,DAY2..."""
import sys, statistics as st
sys.path.insert(0, 'tools/scripts')
import data
h1, h2 = map(int, sys.argv[1].split('-'))
days = sys.argv[2].split(',')
M = data.load('data/NIEDERHORN')
tg = {s: ms for s, ms in M.items() if not data.is_reference(ms) and sum(m['corr_x'] is not None for m in ms) > 500}
noise = {}
ser = {}
for s, ms in tg.items():
for c in ('corr_x', 'corr_y'):
v = {m['t'].replace(second=0, microsecond=0): m[c] for m in ms}
vv = [x for x in v.values() if x is not None]
noise[s, c] = st.median(abs(vv[i + 1] - vv[i]) for i in range(len(vv) - 1))
ser[s, c] = v
from datetime import timedelta
for d in days:
for h in range(h1, h2 + 1):
t = data.parse_time(f'2026-10-{d}T{h:02d}:00')
p = t - timedelta(hours=1)
st_ = []
big = []
for (s, c), v in ser.items():
if v.get(t) is not None and v.get(p) is not None:
z = (v[t] - v[p]) / noise[s, c]
st_.append(abs(z))
if abs(z) > 4: big.append(f"{s.replace('PWD_','').replace('_C09','')}{c[-1]}{z:+.0f}")
print(f'{d}T{h:02d}', f'med|step| {st.median(st_):.1f}' if st_ else '-', len(big), ' '.join(big[:12]))