auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df979301b5
commit
2a5d87659b
279 files changed
+10758
-710
No files matched your search
@@ -0,0 +1,166 @@
|
||||
"""Build labels.jsonl from work/existing.jsonl: kept labels unchanged (review kept, id dropped), corrected ones
|
||||
rewritten (review dropped), added ones appended. Run from the repository root."""
|
||||
import json
|
||||
|
||||
D = "labeling/auto/analyses/NIEDERHORN_2026-10-01T0000_2026-10-11T0500/"
|
||||
old = {json.loads(l)["id"]: json.loads(l) for l in open(D + "work/existing.jsonl")}
|
||||
|
||||
SUN = ("Cause seen on the {img} image of that day: the rising sun is in the field, just above the road of the references, "
|
||||
"with halo, rays and reflections over the whole image; it comes back at the same hours on clear days (not on the overcast 7 October). "
|
||||
"The structure does not move: do not trust the curves over this measurement.")
|
||||
|
||||
C = { # id -> fields that change
|
||||
"NIEDERHORN-12": dict(end="2026-10-01T17:00Z", description=(
|
||||
"At night and in the morning, for several hours: the recognition error of many targets rises, many are rejected at times "
|
||||
"(mostly PWD_IMP_B07_018_C09, PWD_RMP_B15_019_C09, PWD_RMP_B12_018_C09, e09, PWD_IMP_B07_017_C09 and PWD_RMP_B12_017_C09, "
|
||||
"once or twice PWD_RMP_B12_019_C09, PWD_RMP_B15_018_C09, e02, e06, e10 and PWD_RMP_B16_026_C09) and the corr_x of the targets "
|
||||
"on the left dips clearly (PWD_RMP_B15_020_C09, e15, e10, e14, e13, PWD_RMP_B16_026_C09, PWD_IMP_B08_025_C09); the references hold, "
|
||||
"higher up on the road. Gradual return in late morning: PWD_IMP_B07_018_C09, rejected until 15:00, PWD_RMP_B17_026_C09 and e09 "
|
||||
"still stand away from their level at 16:00, then come back. Cause seen on the 07:00 image: a noisy background, "
|
||||
"faint out-of-focus drops on the glass, weaker spots, nothing in front (according to the expert, a rainy night at this station). "
|
||||
"The structure does not move: do not trust the curves over the range. PWD_IMP_B07_016_C09, moved by the foreground plant during "
|
||||
"the same hours, is labelled apart.")),
|
||||
"NIEDERHORN-13": dict(description=(
|
||||
"A single measurement: e02, e06, e09, e10, PWD_RMP_B16_026_C09, PWD_RMP_B17_026_C09, PWD_IMP_B08_025_C09, e14, e15, "
|
||||
"PWD_RMP_B15_020_C09, PWD_IMP_B07_018_C09, PWD_RMP_B15_019_C09, PWD_RMP_B15_018_C09 are rejected and missing from the corrected "
|
||||
"curves; the references stay intact. Kept under the threshold, PWD_RMP_B12_019_C09, PWD_RMP_B12_020_C09, PWD_RMP_B12_018_C09 "
|
||||
"and PWD_RMP_B12_017_C09 make a clear one-measurement jump in corr_y, then return. Cause seen: a freight train, wagon numbers "
|
||||
"legible, on the 12:00 image. Everything returns at the next measurement, after the missing one: the structure does not move, "
|
||||
"point to ignore.")),
|
||||
"NIEDERHORN-16": dict(start="2026-10-02T01:00Z", end="2026-10-02T15:00Z", description=(
|
||||
"At night and in the morning, for several hours: the recognition error of many targets rises, many are rejected for hours "
|
||||
"(PWD_IMP_B07_017_C09, PWD_IMP_B07_018_C09, PWD_RMP_B12_017_C09, PWD_RMP_B12_018_C09, PWD_RMP_B15_018_C09, e09, "
|
||||
"PWD_RMP_B12_019_C09, PWD_RMP_B15_019_C09, e02, e01, at times e06 and PWD_RMP_B16_026_C09) and the corr_x of the targets on the "
|
||||
"left dips clearly (e15, e10, PWD_RMP_B15_020_C09, e14, PWD_IMP_B08_025_C09, PWD_RMP_B16_026_C09), PWD_RMP_B15_019_C09 also rising "
|
||||
"in corr_y; the references hold, higher up on the road. It starts with a first jump of e14 and PWD_IMP_B07_018_C09; gradual "
|
||||
"return in late morning, PWD_RMP_B15_020_C09 and e09 last, their recognition error still high just after the gap. Cause seen on "
|
||||
"the 07:00 image: large out-of-focus drops on the glass and streaks of rain lit by the floodlight, weaker spots (according to "
|
||||
"the expert, a rainy night at this station). The structure does not move: do not trust the curves over the range. "
|
||||
"PWD_IMP_B07_016_C09, moved by the foreground plant, is labelled apart.")),
|
||||
"NIEDERHORN-17": dict(start="2026-10-02T04:00Z", description=(
|
||||
"PWD_IMP_B07_016_C09 alone, from the end of the night, for a good part of the day and the start of the night: its corr_x and "
|
||||
"corr_y jump, then its corr_x makes large peaks, alone among all the targets in early morning, then its measurement is rejected "
|
||||
"at times, or, just under the threshold, its corr_x or corr_y moves away for one measurement and returns, its recognition error "
|
||||
"dropping when it moves; its neighbors do not move outside the rainy night of the morning. Cause seen on the 05:00, 08:00, 09:00 "
|
||||
"and 10:00 thumbnails: the leaves of the foreground plant, pushed by the wind, touch then cover the spot of the target. "
|
||||
"The structure does not move: ignore the curve of this target over the range.")),
|
||||
"NIEDERHORN-22": dict(description=(
|
||||
"A single measurement, in the middle of the night: e01 and e02, side by side at the left edge of the image, are rejected "
|
||||
"together and missing from their corrected curves; their neighbors and the references do not move. On the 10:00 thumbnails, "
|
||||
"the spots of e01 and e02 have both disappeared, a small spot next to e01 too, while the night before and after they are "
|
||||
"intact: something hides them; nothing clear on the whole image, cause not established, a signature close to that of the "
|
||||
"trains on the far track that hide the targets on the left. Return at the next measurement: the structure does not move, "
|
||||
"point to ignore.")),
|
||||
"NIEDERHORN-23": dict(description=(
|
||||
"At sunrise, for a single measurement: the recognition error of the references rises, strongly for PWD_REF10_C09, clearly for "
|
||||
"PWD_REF11_C09 and PWD_REF08_C09, a little for PWD_REF09_C09, without rejection; the compensation is slightly disturbed and the "
|
||||
"corr_x of PWD_IMP_B07_018_C09 and e02 jumps, that of PWD_IMP_B07_016_C09 and the corr_y of e13 a little, then they come back. "
|
||||
+ SUN.format(img="13:00"))),
|
||||
"NIEDERHORN-25": dict(description=(
|
||||
"At sunrise, for a single measurement: the recognition error of the references rises, strongly for PWD_REF10_C09, clearly for "
|
||||
"PWD_REF11_C09, a little for PWD_REF08_C09, without rejection; the compensation is disturbed and the corr_x of "
|
||||
"PWD_IMP_B07_018_C09 jumps strongly, that of PWD_RMP_B12_019_C09, PWD_RMP_B12_018_C09 and PWD_IMP_B07_017_C09 clearly, the "
|
||||
"corr_y of e10, e13, e15 and PWD_RMP_B15_020_C09 too, then they come back. " + SUN.format(img="13:00"))),
|
||||
"NIEDERHORN-27": dict(description=(
|
||||
"At sunrise, for a single measurement: the recognition error of the references rises, clearly for PWD_REF10_C09 and "
|
||||
"PWD_REF08_C09, a little for PWD_REF09_C09 and PWD_REF11_C09, without rejection; the compensation is only slightly disturbed: "
|
||||
"the references and PWD_RMP_B15_018_C09 move a little in corr_x or corr_y, the other targets hardly. " + SUN.format(img="13:00"))),
|
||||
"NIEDERHORN-31": dict(description=(
|
||||
"A single measurement: e02, e06, e09, e10, PWD_RMP_B16_026_C09, PWD_RMP_B17_026_C09, PWD_IMP_B08_025_C09, e14, e15, "
|
||||
"PWD_RMP_B15_020_C09, PWD_IMP_B07_018_C09, PWD_RMP_B15_019_C09, PWD_RMP_B15_018_C09 are rejected and missing from the corrected "
|
||||
"curves; the references stay intact. Remaining under the threshold, PWD_RMP_B12_019_C09, PWD_RMP_B12_020_C09, "
|
||||
"PWD_RMP_B12_018_C09, e13 and PWD_RMP_B12_017_C09 make a clear one-measurement jump in corr_y, PWD_IMP_B07_017_C09 a smaller one "
|
||||
"in corr_x, then come back (the rejection of PWD_RMP_B12_019_C09 at the measurement before is labelled apart). Cause seen: a "
|
||||
"dark band with a right edge cutting across the scene on the 12:00 image, a train in the dim dawn light. Everything comes back "
|
||||
"at the next measurement: the structure does not move, point to ignore.")),
|
||||
"NIEDERHORN-37": dict(end="2026-10-07T23:00Z", description=(
|
||||
"PWD_IMP_B07_016_C09 alone, until the evening: at times its recognition error jumps almost to the maximum and the measurement is "
|
||||
"rejected, or, just under the threshold, its corr_x or corr_y makes a one-measurement spike, then comes back; the last spikes, "
|
||||
"in the evening, come with its recognition error dropping; its neighbours do not move. Cause seen on the 14:00 thumbnail: a "
|
||||
"blurred leaf covers the spot, which can no longer be seen; on the 19:00, 20:00 and 22:00 thumbnails the leaves still cross the "
|
||||
"spot; it is the foreground plant, shaken by the wind, which, according to the expert, passes in front of this target. "
|
||||
"The structure does not move: ignore this target's curve over the range.")),
|
||||
"NIEDERHORN-41": dict(description=(
|
||||
"At sunrise, for a single measurement: the recognition error of the references rises, strongly for PWD_REF10_C09, clearly for "
|
||||
"PWD_REF08_C09 and PWD_REF11_C09, a little for PWD_REF09_C09, without rejection; the compensation is disturbed: PWD_REF11_C09 "
|
||||
"and PWD_REF12_C09 move apart in corr_x, the corr_x of e02, e10 and PWD_IMP_B07_017_C09 jumps, then they come back. "
|
||||
+ SUN.format(img="13:00"))),
|
||||
"NIEDERHORN-45": dict(description=(
|
||||
"At sunrise, for a single measurement: the recognition error of the references rises, strongly for PWD_REF10_C09, clearly for "
|
||||
"PWD_REF08_C09, PWD_REF09_C09 and PWD_REF11_C09, without rejection; the compensation is disturbed: PWD_REF11_C09 and "
|
||||
"PWD_REF12_C09 move apart in corr_x and the corr_x of the targets on the left jumps together, strongly for e02, clearly for "
|
||||
"e06, e10, e13, e14 and e15, while that of PWD_IMP_B07_017_C09 and PWD_IMP_B07_018_C09 dips, then they come back. "
|
||||
+ SUN.format(img="13:00"))),
|
||||
"NIEDERHORN-49": dict(end="2026-10-10T02:00Z", description=(
|
||||
"PWD_IMP_B07_016_C09 alone: at times its recognition error jumps almost to the maximum and the measurement is rejected, or, just "
|
||||
"under the threshold, its corr_x or corr_y makes a one-measurement spike, then comes back; its neighbours do not move. Cause "
|
||||
"seen on the 23:00 thumbnail: leaves touch the spot; on the 00:00 and 01:00 thumbnails they still touch it, its corr_y away "
|
||||
"and its recognition error at its lowest; it is the foreground plant, shaken by the wind, which, according to the expert, "
|
||||
"passes in front of this target. The structure does not move: ignore this target's curve over the range.")),
|
||||
}
|
||||
|
||||
PLANT = ("Cause seen on the {img} thumbnail: {what}; it is the foreground plant, shaken by the wind, which, according to the expert, "
|
||||
"passes in front of this target. The structure does not move: ignore this target's curve over the range.")
|
||||
A = [
|
||||
dict(start="2026-10-01T02:00Z", end="2026-10-01T03:00Z", family="Obstruction", description=(
|
||||
"PWD_IMP_B07_016_C09 alone, at the first measurement after the missing one, in the night: its corr_x and corr_y jump clearly "
|
||||
"for one measurement, without rejection, its recognition error a little lower, then come back; its neighbours and the "
|
||||
"references do not move. " + PLANT.format(img="02:00", what="the tip of a leaf touches the spot"))),
|
||||
dict(start="2026-10-01T05:00Z", end="2026-10-01T16:00Z", family="Obstruction", description=(
|
||||
"PWD_IMP_B07_016_C09, during the rainy night and morning: its corr_x keeps jumping clearly from one measurement to the next, up "
|
||||
"and down, its corr_y at times too, much more than its neighbours, without rejection, its recognition error dropping when it "
|
||||
"jumps. " + PLANT.format(img="14:00", what="a leaf touches the spot") + " The rain of the same hours is labelled apart.")),
|
||||
dict(start="2026-10-01T19:00Z", end="2026-10-01T21:00Z", family="Displacement", description=(
|
||||
"For a single measurement, under an overcast day: the corr_y of e10 jumps clearly up, its raw position moving in the image and "
|
||||
"its recognition error rising a little, while the corr_x of PWD_IMP_B08_025_C09 dips clearly, its recognition error dropping; "
|
||||
"both come back at the next measurement. The references, the camera and the other targets stay calm. Nothing in front of the "
|
||||
"targets on the 20:00 image, nor on the thumbnails of e10 and PWD_IMP_B08_025_C09 before, during and after. Cause not "
|
||||
"established (a passage too brief for the image, or a reflection, possible). Not a lasting movement: point to ignore.")),
|
||||
dict(start="2026-10-01T23:00Z", end="2026-10-02T01:00Z", family="Displacement", description=(
|
||||
"For a single measurement, in the night: the corr_x of PWD_IMP_B08_025_C09 alone dips clearly, its recognition error dropping "
|
||||
"as at 20:00 the same evening, then comes back; its neighbours and the references do not move. Nothing in front of its spot on "
|
||||
"the 00:00 thumbnail. Cause not established. Not a lasting movement: point to ignore.")),
|
||||
dict(start="2026-10-02T01:00Z", end="2026-10-02T03:00Z", family="Obstruction", description=(
|
||||
"PWD_IMP_B07_016_C09, in the night, for a single measurement: its corr_y jumps strongly and its corr_x clearly, without "
|
||||
"rejection, its recognition error dropping, then they come back; the rain that starts at the same measurement touches its "
|
||||
"neighbours much less. " + PLANT.format(img="02:00", what="leaves touch the spot")
|
||||
+ " The plant comes back two measurements later, labelled apart.")),
|
||||
dict(start="2026-10-04T17:00Z", end="2026-10-04T19:00Z", family="Obstruction", description=(
|
||||
"PWD_IMP_B07_016_C09 alone, for a single measurement during the warm hours: its corr_x jumps clearly and its corr_y drops, "
|
||||
"without rejection, its recognition error dropping, then they come back; its neighbours only follow the thermal cycle of the "
|
||||
"day. " + PLANT.format(img="18:00", what="blurred leaves cross the spot"))),
|
||||
dict(start="2026-10-06T10:00Z", end="2026-10-06T12:00Z", family="Visibility", description=(
|
||||
"At dawn, a single measurement: PWD_RMP_B12_019_C09 is rejected and missing from its corrected curve: its recognition error, "
|
||||
"just under its threshold all night, goes just above it, without a jump; its neighbours and the references do not move. On "
|
||||
"the 11:00 thumbnail, the spot is intact and smaller, in a darker image, nothing in front. Cause deduced: a target at the "
|
||||
"limit of its threshold that crosses it when the light changes, not a passage. The train of the next measurement is labelled "
|
||||
"apart. The structure does not move: point to ignore.")),
|
||||
dict(start="2026-10-08T11:00Z", end="2026-10-08T13:00Z", family="Displacement", description=(
|
||||
"At dawn, a single measurement: the corr_y of PWD_RMP_B16_026_C09 alone jumps clearly up, without rejection and without its "
|
||||
"recognition error moving, then comes back; its neighbours, the references and the camera do not move. On the 12:00 thumbnail "
|
||||
"the image is very dark, between the night lighting and the day, the spot smaller but present, nothing in front. Cause not "
|
||||
"established (the change of light is possible). Not a lasting movement: point to ignore.")),
|
||||
dict(start="2026-10-08T21:00Z", end="2026-10-08T22:00Z", family="Obstruction", description=(
|
||||
"PWD_IMP_B07_016_C09 alone, at the first measurement after the missing one: its corr_y jumps strongly for one measurement, "
|
||||
"without rejection, then comes back; its neighbours and the references do not move. "
|
||||
+ PLANT.format(img="21:00", what="a leaf lies on the spot"))),
|
||||
dict(start="2026-10-10T12:00Z", end="2026-10-10T14:00Z", family="Visibility", description=(
|
||||
"At sunrise, behind a haze: the recognition error of PWD_REF09_C09 rises clearly at 12:00, that of PWD_REF10_C09 and "
|
||||
"PWD_REF11_C09 a little at 13:00, without rejection; at 13:00 the corr_x of PWD_IMP_B08_025_C09 and e10 jumps clearly for one "
|
||||
"measurement, then comes back. Cause seen on the 12:00 and 13:00 images of that day: the sun glows behind the buildings then "
|
||||
"in the haze at the top of the field, the scene darkened against it; the same nuisance as on the clear mornings, weaker. "
|
||||
"The structure does not move: do not trust the curves over this measurement.")),
|
||||
]
|
||||
|
||||
out = []
|
||||
for i, d in old.items():
|
||||
if i in C:
|
||||
d = {k: v for k, v in d.items() if k not in ("review",)}
|
||||
d.update(C[i])
|
||||
out.append({k: v for k, v in d.items() if k != "id"})
|
||||
out += A
|
||||
out.sort(key=lambda d: (d["start"], d["end"]))
|
||||
with open(D + "labels.jsonl", "w") as f:
|
||||
for d in out:
|
||||
f.write(json.dumps(d, ensure_ascii=False) + "\n")
|
||||
print(len(out), "labels;", len(old) - len(C), "kept,", len(C), "corrected,", len(A), "added")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Per instant, sensors whose corr deviates from their level in a reference window by more than K noises (r: rejected).
|
||||
Usage: python3 dev.py data/NIEDERHORN START END REF_START REF_END [K] [sensors,comma]"""
|
||||
import sys, statistics
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
f, a, b, ra, rb = sys.argv[1], *map(data.parse_time, sys.argv[2:6])
|
||||
K = float(sys.argv[6]) if len(sys.argv) > 6 else 5
|
||||
only = set(sys.argv[7].split(",")) if len(sys.argv) > 7 else None
|
||||
PERM = {"e03", "e04", "e05", "e07", "e08", "e11", "e12"}
|
||||
M = data.load(f)
|
||||
for v in M.values():
|
||||
for m in v: m["t"] = m["t"].replace(second=0, microsecond=0)
|
||||
lvl, noise = {}, {}
|
||||
for s, v in M.items():
|
||||
for c in ("corr_x", "corr_y"):
|
||||
r = [m[c] for m in v if ra <= m["t"] <= rb and m[c] is not None]
|
||||
allv = [m[c] for m in v if m[c] is not None]
|
||||
d = [abs(y - x) for x, y in zip(allv, allv[1:])]
|
||||
lvl[s, c] = statistics.median(r) if r else None
|
||||
noise[s, c] = statistics.median(d) if d else 0
|
||||
times = sorted({m["t"] for v in M.values() for m in v if a <= m["t"] <= b})
|
||||
by = {s: {m["t"]: m for m in v} for s, v in M.items()}
|
||||
short = lambda s: s.replace("PWD_", "").replace("_C09", "")
|
||||
for t in times:
|
||||
out = []
|
||||
for s in sorted(M):
|
||||
if s in PERM or (only and s not in only): continue
|
||||
m = by[s].get(t)
|
||||
if m is None: continue
|
||||
if data.rejected(m): out.append(f"{short(s)}:r"); continue
|
||||
for c in ("corr_x", "corr_y"):
|
||||
if lvl[s, c] is None or not noise[s, c]: continue
|
||||
d = (m[c] - lvl[s, c]) / noise[s, c]
|
||||
if abs(d) >= K: out.append(f"{short(s)}:{c[-1]}{d:+.0f}")
|
||||
print(data.iso(t), len(out), " ".join(out))
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Per instant: pose, temperature, rejected sensors (outside the permanent ones), one-measurement spikes of corr.
|
||||
Usage: python3 instants.py data/NIEDERHORN START END [K]"""
|
||||
import sys, statistics
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
from datetime import timedelta
|
||||
folder, start, end = sys.argv[1], data.parse_time(sys.argv[2]), data.parse_time(sys.argv[3])
|
||||
K = float(sys.argv[4]) if len(sys.argv) > 4 else 5
|
||||
PERM = {"e03", "e04", "e05", "e07", "e08", "e11", "e12"}
|
||||
M = data.load(folder)
|
||||
for v in M.values():
|
||||
for m in v: m["t"] = m["t"].replace(second=0, microsecond=0)
|
||||
lo, hi = start - timedelta(hours=24), end + timedelta(hours=24)
|
||||
M = {s: [m for m in v if lo <= m["t"] <= hi] for s, v in M.items()}
|
||||
refs = {s for s, v in M.items() if data.is_reference(v)}
|
||||
cam = {p["t"]: p for p in data.load_camera(folder)}
|
||||
temp = dict(data.load_temperature(folder))
|
||||
noise = {}
|
||||
for s, v in M.items():
|
||||
for c in ("corr_x", "corr_y"):
|
||||
vals = [m[c] for m in v if m[c] is not None]
|
||||
d = [abs(b - a) for a, b in zip(vals, vals[1:])]
|
||||
noise[s, c] = statistics.median(d) if d else 0
|
||||
times = sorted({m["t"] for v in M.values() for m in v if start <= m["t"] <= end})
|
||||
idx = {s: {m["t"]: i for i, m in enumerate(v)} for s, v in M.items()}
|
||||
prev = None
|
||||
for t in times:
|
||||
p = cam.get(t, {})
|
||||
th = t.replace(minute=0)
|
||||
tc = temp.get(th)
|
||||
rej = [s for s in sorted(M) if s not in PERM and t in idx[s] and data.rejected(M[s][idx[s][t]])]
|
||||
spikes = []
|
||||
for s in sorted(M):
|
||||
if s in PERM or t not in idx[s]: continue
|
||||
i = idx[s][t]; v = M[s]
|
||||
for c in ("corr_x", "corr_y"):
|
||||
x = v[i][c]
|
||||
if x is None or noise[s, c] == 0: continue
|
||||
nb = [v[j][c] for j in (i - 1, i + 1) if 0 <= j < len(v) and v[j][c] is not None]
|
||||
if not nb: continue
|
||||
dev = (x - statistics.mean(nb)) / noise[s, c]
|
||||
if abs(dev) >= K: spikes.append(f"{s.replace('PWD_','').replace('_C09','')}:{c[-1]}{dev:+.0f}")
|
||||
gap = "" if prev is None or t - prev <= timedelta(minutes=61) else f" GAP({int((t-prev).total_seconds()//3600)-1})"
|
||||
prev = t
|
||||
f = lambda k, n: "" if p.get(k) is None else f"{p[k]:.{n}f}"
|
||||
print(f"{data.iso(t)}{gap} T={tc if tc is None else round(tc,1)} pan={f('pan',4)} tilt={f('tilt',4)} rot={f('rotation',4)} sc={f('scale',5)} s0={f('sigma0',4)}"
|
||||
f" | rej[{len(rej)}]: {','.join(r.replace('PWD_','').replace('_C09','') for r in rej)} | spikes: {' '.join(spikes)}")
|
||||
Reference in new issue
Block a user