auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df979301b5
commit
2a5d87659b
279 files changed
+10758
-710
No files matched your search
@@ -0,0 +1,43 @@
|
||||
"""Builds labels.jsonl from work/existing.jsonl: kept labels unchanged (review kept), corrected ones rewritten
|
||||
(review dropped), new ones added; no id field. Run from the repository root."""
|
||||
import json
|
||||
|
||||
A = "labeling/auto/analyses/NIEDERHORN_2026-10-11T0500_2026-10-21T1200/"
|
||||
old = {d["id"]: d for d in map(json.loads, open(A + "work/existing.jsonl"))}
|
||||
|
||||
SUN = ("Cause seen on the image of {day} at 13:00: the rising sun enters the field just above the road of the references, "
|
||||
"with halo and rays across the upper part of the image; it comes back at the same hours on clear days. "
|
||||
"The structure does not move: do not trust the curves on these measurements.")
|
||||
|
||||
fix = {
|
||||
"NIEDERHORN-53": {"description": "One measurement is missing for all the sensors, references and targets: one point absent on all the curves; there is no image at that hour either. The station is running (metrics present at that hour, normal voltage before and after) and no other station misses this measurement; the cause of the gap is not established (measurement not made or not transmitted). The curves resume at the next measurement without a jump: nothing to do."},
|
||||
"NIEDERHORN-55": {"end": "2026-10-12T06:00Z", "description": "In the evening and through the night of 11 to 12.10, for several hours: the recognition error of many targets rises, several are rejected at times (especially PWD_RMP_B12_019_C09, e06, e09, PWD_RMP_B15_018_C09, PWD_RMP_B16_026_C09, PWD_IMP_B07_017_C09, PWD_IMP_B07_018_C09, PWD_RMP_B12_017_C09 and PWD_RMP_B12_018_C09) and the others fluctuate; just under the threshold, the corr_x of PWD_RMP_B15_020_C09, PWD_IMP_B08_025_C09, e10, e15 and PWD_RMP_B16_026_C09 drops strongly at the worst of it; the references hold, higher up on the road. Worst in the middle of the night, then a gradual return: after the train of 04:00 (labelled apart), PWD_RMP_B15_020_C09 and PWD_IMP_B08_025_C09 are still off their level at 05:00 and back at 06:00. Probable cause: rain or mist (smaller and weaker spots on the 01:00 and 03:00 thumbnails than on a clear night, without any obstacle). The structure does not move: do not trust the curves over the range."},
|
||||
"NIEDERHORN-63": {"end": "2026-10-14T06:00Z", "description": "Thermal cycle of the day (13/10). The housing temperature rises strongly from the morning until 18:00, fastest between 13:00 and 14:00, where it gains markedly, then falls all night. The camera pose follows: from 14:00 to 16:00, the tilt rises markedly and the rotation drops markedly, then both come back at once at 17:00; the tilt then drops markedly below its morning level at 19:00 and only comes back by 05:00; the pan rises markedly until 17:00, then goes back down below its morning level, lowest around 00:00, and comes back slowly during the night; the scale rises strongly until 20:00 and only comes back slowly, during the night. The raw positions of all the sensors, references included, move together in the image; the compensation follows poorly: the corr_x of PWD_IMP_B07_018_C09, PWD_RMP_B12_018_C09, PWD_IMP_B07_017_C09 and PWD_RMP_B12_017_C09 in the centre and on the right drops strongly, that of e02 and PWD_RMP_B15_018_C09 rises markedly; the corr_y moves in the opposite direction, less strongly: it rises for PWD_RMP_B12_017_C09 and PWD_IMP_B07_017_C09, it drops for PWD_RMP_B12_019_C09, PWD_RMP_B12_018_C09 and PWD_RMP_B12_020_C09. From one measurement to the next, as long as the housing is hot (13:00-22:00), the pose and the curves also fluctuate: the step from one measurement to the next is far larger than at night on the pan, the tilt and the rotation, markedly larger on the corr_x of the targets; the recognition error of the targets, however, does not rise (it drops a little). The curves come back to their night level only in the night, around 05:00, PWD_IMP_B07_018_C09 and PWD_RMP_B12_018_C09 last. Deduced cause: the camera and its mount move with the housing temperature (pose and temperature go together), the fixed references on the road move with the targets in the image. Normal, nothing to do."},
|
||||
"NIEDERHORN-73": {"description": "After the gap of the night of 15 to 16.10, the targets on the left edge e06, e09, e10, e13, e15 and PWD_RMP_B16_026_C09 resume at another level: corr_x markedly lower for e06, e09, e10, e13 and PWD_RMP_B16_026_C09, higher for e15, corr_y higher for e09, e10, e13 and e15; they stay there in the following weeks, with their cycle each day. At the same time, the recognition error of e06, e13 and PWD_RMP_B16_026_C09 goes up a level and stays there, e06 close to its threshold. The other targets and the references do not change level; the raw positions of these targets depart from the overall movement and stay offset. On the 14:00 thumbnail of 16.10 the spot of e06 looks split and that of e13 carries an appendage, and on the night images of the following days the spots of this left part look smeared, two neighbours merging, which they did not before 15.10; the night before, at 02:00 on 15.10, a first smaller step of some of these targets came with a faint haze over this part of the image (labelled apart). Cause not established: something close to the lens over the left part of the image (a spider web or a deposit) is possible; a local movement of this part of the tracks or an intervention during the gap is not excluded. To check: look at the lens and at these targets on site, monitor them."},
|
||||
"NIEDERHORN-74": {"start": "2026-10-16T11:00Z"},
|
||||
"NIEDERHORN-75": {"end": "2026-10-17T12:00Z", "description": "Weak, short warming then a long cooling, when measurements resume after the night gap (NIEDERHORN#72). The housing temperature rises a little from 12:00 to 15:00, then falls strongly all evening and all night, until 12:00 on 17.10, far lower than on the previous nights. The camera scale rises a little with it until 15:00, then drops steadily all night, far below its level of the previous nights; the pan drops a little until 13:00, then rises steadily all evening and night, markedly; the tilt and the rotation do not move. In the morning the corrected curves show no readable cycle (the disturbance after the gap, NIEDERHORN#74, and the new level of the left-edge targets, NIEDERHORN#73); during the night, the compensation follows the slow drift poorly: the corr_x of PWD_RMP_B12_018_C09 and PWD_RMP_B12_020_C09 rises markedly, that of PWD_IMP_B08_025_C09 and e10 drops; the corr_y of e09, e06 and PWD_RMP_B12_018_C09 rises markedly, that of PWD_RMP_B12_017_C09 drops a little. The drift lasts until the start of the cycle of 17.10 (labelled apart), which brings the pose back; at equal housing temperature the pose is back at the same place. Deduced cause: the camera and its mount move with the housing temperature. Normal, nothing to do."},
|
||||
"NIEDERHORN-81": {"description": "At sunrise, over a few measurements: the recognition error of the references PWD_REF08_C09 and PWD_REF09_C09 rises markedly from 12:00, that of PWD_REF10_C09 and PWD_REF11_C09 a little at 13:00, without rejection, and only comes back slowly over the morning; the compensation is slightly disturbed and the corr_x, corr_y of many targets jump or fluctuate; e09 and e02, on the left, are rejected at 13:00. " + SUN.format(day="17.10")},
|
||||
"NIEDERHORN-84": {"end": "2026-10-18T15:00Z", "description": "At sunrise, over a few measurements: the recognition error of the reference PWD_REF09_C09 jumps at 12:00, those of PWD_REF08_C09, PWD_REF10_C09 and PWD_REF11_C09 rise markedly at 13:00 and are still above their level at 14:00, without rejection; the compensation is slightly disturbed and the corr_x, corr_y of many targets jump or fluctuate, the references PWD_REF11_C09 and PWD_REF12_C09 moving apart on corr_x at 13:00 and 14:00; e06 and e09, on the left, are rejected at 13:00. " + SUN.format(day="18.10")},
|
||||
"NIEDERHORN-89": {"description": "At sunrise, over a few measurements: the recognition error of the reference PWD_REF09_C09 jumps at 12:00, those of PWD_REF08_C09, PWD_REF10_C09 and PWD_REF11_C09 rise markedly at 13:00 and are still above their level at 14:00, without rejection; the compensation is disturbed: at 12:00 the corr_x of most targets on the left of the image drops together for one measurement, then the corr_x, corr_y of many targets jump or fluctuate; e06, on the left, is rejected at 13:00 and 14:00, e09 at 13:00. " + SUN.format(day="19.10")},
|
||||
"NIEDERHORN-94": {"end": "2026-10-20T15:00Z", "description": "At sunrise, over a few measurements: the recognition error of the references PWD_REF10_C09 and PWD_REF11_C09 rises markedly at 13:00, that of PWD_REF08_C09 a little, and is still above its level at 14:00, without rejection; the compensation is slightly disturbed and the corr_x, corr_y of many targets jump or fluctuate, the references PWD_REF11_C09 and PWD_REF12_C09 moving apart on corr_x at 13:00 and 14:00; e06 and e09, on the left, are rejected at 13:00 (at 12:00 they are hidden by the train labelled apart). " + SUN.format(day="20.10")},
|
||||
}
|
||||
|
||||
new = [
|
||||
{"start": "2026-10-15T01:00Z", "end": "2026-10-15T03:00Z", "family": "Obstruction", "description": "A single measurement, at night: e01 is rejected and missing from its corrected curve; remaining under the threshold, e09 makes a one-measurement jump on corr_x and corr_y, e10 and PWD_RMP_B16_026_C09 jump on corr_x beyond the level where they then stay, and the corr_y of the references PWD_REF11_C09, PWD_REF12_C09 and PWD_REF09_C09 jumps a little, then comes back. Cause seen on the 02:00 image: lit flying insects pass very close to the lens, as large blurred pale spots, one over the left group of targets and others near the references; there are none on the 03:00 image. Everything but the step labelled apart comes back at the next measurement: the structure does not move, point to ignore."},
|
||||
{"start": "2026-10-15T01:00Z", "end": "2026-10-15T04:00Z", "family": "Displacement", "description": "At the 02:00 measurement, at night, PWD_RMP_B16_026_C09 steps up on corr_y, clearly, and on corr_x, a little, and stays at the new level; e10 and e13 shift a little the same way and stay. At the same measurement, the recognition error of e09, e10 and PWD_RMP_B16_026_C09 goes up a level, that of e13 a little, and stays there until the gap of the following night (NIEDERHORN#72). The other targets and the references keep their level. On the night images, a faint haze surrounds the spots of e09, e10 and PWD_RMP_B16_026_C09 at 02:00, 03:00 and 05:00, absent the night before; at 02:00 lit insects fly in front of the lens (labelled apart). After the gap, the same left-edge targets change level again (NIEDERHORN#73). Cause not established: something left close to the lens over this part of the image (an insect, a spider web) is possible; a movement of these targets is not excluded. To check: look at the lens and at these targets, monitor them."},
|
||||
]
|
||||
|
||||
out = []
|
||||
for i, d in old.items():
|
||||
d = dict(d)
|
||||
del d["id"]
|
||||
if i in fix:
|
||||
d.update(fix[i])
|
||||
d.pop("review", None)
|
||||
out.append(d)
|
||||
out += new
|
||||
out.sort(key=lambda d: (d["start"], d["end"]))
|
||||
with open(A + "labels.jsonl", "w") as f:
|
||||
for d in out:
|
||||
f.write(json.dumps(d, ensure_ascii=False) + "\n")
|
||||
print(len(out), "labels;", len(old) - len(fix), "kept,", len(fix), "corrected,", len(new), "added")
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Per instant: targets whose corr_x / corr_y jump against their local level (median of the 3 measurements on each side,
|
||||
without the point), rejected sensors outside the always-rejected ones, and the labels covering the instant.
|
||||
Usage: python3 scan.py START END [K]"""
|
||||
import json, statistics, sys
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
|
||||
A = "labeling/auto/analyses/NIEDERHORN_2026-10-11T0500_2026-10-21T1200/work/"
|
||||
start, end = data.parse_time(sys.argv[1]), data.parse_time(sys.argv[2])
|
||||
K = float(sys.argv[3]) if len(sys.argv) > 3 else 5
|
||||
ALWAYS = {"e03", "e04", "e05", "e07", "e08", "e11", "e12"}
|
||||
by = data.load("data/NIEDERHORN")
|
||||
labels = [json.loads(l) for f in ("existing.jsonl", "spanning.jsonl") for l in open(A + f)]
|
||||
for l in labels:
|
||||
l["s"], l["e"] = data.parse_time(l["start"]), data.parse_time(l["end"])
|
||||
|
||||
dev = {} # t -> list of strings
|
||||
rej = {}
|
||||
for s, ms in by.items():
|
||||
if s in ALWAYS:
|
||||
continue
|
||||
ms = [m for m in ms if start - __import__("datetime").timedelta(hours=6) <= m["t"] <= end]
|
||||
for c in ("corr_x", "corr_y"):
|
||||
v = [(m["t"], m[c]) for m in ms if m[c] is not None]
|
||||
if len(v) < 8:
|
||||
continue
|
||||
diffs = [abs(v[i + 1][1] - v[i][1]) for i in range(len(v) - 1)]
|
||||
noise = statistics.median(diffs) or 1e-9
|
||||
for i, (t, x) in enumerate(v):
|
||||
nb = [y for _, y in v[max(0, i - 3):i] + v[i + 1:i + 4]]
|
||||
d = (x - statistics.median(nb)) / noise
|
||||
if abs(d) >= K and start <= t <= end:
|
||||
dev.setdefault(t, []).append(f"{s.replace('PWD_','').replace('_C09','')}:{c[-1]}{d:+.0f}")
|
||||
for m in ms:
|
||||
if data.rejected(m) and start <= m["t"] <= end:
|
||||
rej.setdefault(m["t"], []).append(s.replace("PWD_", "").replace("_C09", ""))
|
||||
|
||||
for t in sorted(set(dev) | set(rej)):
|
||||
cov = [l["id"].split("-")[1] + l["family"][:3] for l in labels if l["s"] <= t <= l["e"]]
|
||||
print(data.iso(t), "| rej:", ",".join(rej.get(t, [])), "| dev:", " ".join(dev.get(t, [])), "| lab:", ",".join(cov))
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Values of chosen sensors per instant: corr_x/corr_y/error_value (r = rejected).
|
||||
Usage: python3 values.py START END SENSOR[,SENSOR...]"""
|
||||
import sys
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
|
||||
start, end = data.parse_time(sys.argv[1]), data.parse_time(sys.argv[2])
|
||||
by = data.load("data/NIEDERHORN")
|
||||
names = sys.argv[3].split(",")
|
||||
cols = []
|
||||
for n in names:
|
||||
s = next(k for k in by if k == n or k == f"PWD_{n}_C09")
|
||||
cols.append((s, {m["t"]: m for m in data.filter_window(by[s], start, end)}))
|
||||
times = sorted({t for _, d in cols for t in d})
|
||||
f = lambda v: " -" if v is None else f"{v:+.3f}"
|
||||
print("time " + " | ".join(f"{s.replace('PWD_', '').replace('_C09', ''):^22}" for s, _ in cols))
|
||||
for t in times:
|
||||
row = []
|
||||
for s, d in cols:
|
||||
m = d.get(t)
|
||||
row.append(" none " if m is None else
|
||||
f"{f(m['corr_x'])} {f(m['corr_y'])} {m['error_value']:.2f}{'r' if data.rejected(m) else ' '}")
|
||||
print(data.iso(t), " | ".join(row))
|
||||
Reference in new issue
Block a user