Files
aimsight-detectors/scripts/displacement.py
T

327 lines
21 KiB
Python
Raw Normal View History

"""Detector of the Displacement family (labeling/familles.md): sudden jumps of corr_x / corr_y (px) of each target, at
the native cadence of the "advanced" export, and slow drift (end of file). A flipping target ("Target flipping")
belongs to the Target issue family: episodes() returns it, detect() leaves it to target_issue.py.
1. Noise of the series = median of the gaps from one measurement to the next; jump = measurement K noises or more away
from the level before (median of the BASE previous measurements); the first BASE measurements, lacking a level
before, are compared with the level after (median of the BASE next ones), a gap there is a jump back at once.
2. Back (an hour of measurements in a row within half the jump: BASE at 15 min, a single one at 4 h; wf-improve
MICHIGAN, mod. 1) within a day, or one day later back at its usual place (first day of the export) or at the day
before's: "Sudden jump, back to normal", greyed; its cause is not established here (categories.py gives it the
cause of the category that overlaps it, otherwise: to be checked on the images).
3. Otherwise "Lasting shift", never greyed (maybe a real movement); less than a day of hindsight: "Jump, not back yet",
not greyed either.
4. Jumps following each other within GAP, all targets together, make a single episode (common mode); a single lasting
(or too recent) target is enough not to grey the episode.
5. Global references excluded; rejected measurements (empty corr) skipped.
6. Common peak (NORDEND, batch 01): at the same instant, at least PEAK_MIN targets move away by at least K_PEAK noises
from both neighbouring measurements, on the same side (the whole image shaken for one step): a jump back for each,
even under K. It joins the episodes of point 4 (next to a lasting jump, it greys nothing).
7. (CLARIDEN, batch 01) a jump that never comes back no longer ends the reading of the series, its new level becomes
the level before; an episode that is not greyed ends at its last jump, not at the returns; a flipping target
(REPEAT jumps back or more, each within a day of the previous one) makes a "back to normal" episode on its own, from
the first to the last flip, outside the common episodes. A target that flips for at least FLIP, with at most
MAX_FLIPPING targets at the same time, becomes "Target flipping" (Target issue, D-074); a lasting jump of at least
LARGE of the targets, "Large shift" (everything moved), always to be checked. wf-improve session Target issue, mod. 1,
S2: FLIP from 6 h to 36 h; a target flipping for a day during a nuisance of the whole scene (rain, fog, night of poor
visibility) is not a problem target; its jumps back go to the nuisance overlapping them, otherwise stay
Displacement.
8. Jump under K that stays (NIEDERHORN, batch 01): a jump of at least K_LASTING noises counts if it is lasting in the
sense of point 3 and the median of the day after moves away from that of the day before, on the jump's side, by at
least PLATEAU times the ordinary variation of the level from one UTC day to the next (median of the gaps between
daily levels); otherwise it is ignored (neither back nor too recent: only jumps of K noises or more are).
"""
import statistics
import sys
from datetime import timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent)) # data.py, next to it
import data # noqa: E402
K = 20 # tuned on E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 (17 and 19
# noises) are caught by visibility.py: do not lower K
K_LASTING = 7 # under K, a jump that stays counts from K_LASTING noises (NIEDERHORN, batch 01: e09, corr_y −0.18 px at
# once, 8 noises, stayed 5 days)
PLATEAU = 8 # ... if it stays: median of the day after, on the jump's side, PLATEAU times the ordinary variation of the
# level from one day to the next away from the day before (e09: 13; E-001, healthy targets: 6 at most)
K_PEAK = 6 # one-step peak common to several targets (NORDEND shakes, 6 to 11 noises, batch 01)
PEAK_MIN = 3 # targets peaking at the same step for a common peak (one or two: ordinary noise)
BASE = 4 # measurements that make the level before (1 h at 15 min)
GAP = timedelta(hours=1) # two jumps within an hour, all targets together: a single episode
BACK_H = 24 # back within a day = artefact (E-002: plateaus up to 11.5 h)
FLIP = timedelta(hours=36) # a problem target flips for several days (PR_BG: 2.5 days); less is an agitated morning
# (E-009: C8 and C13, 5 flips in 1 h 20 during the movement) or a night of nuisance (NORDEND, JUNGFRAU:
# 12 to 28 h within a loss of visibility), a jump back, absorbed by its host
MAX_FLIPPING = 2 # targets flipping at the same time, at most (FLORIDA: the two stuck ones; E-002, cobweb on the
# references: 15 at once)
LARGE = 0.5 # share of the targets of a lasting jump for a large shift (E-004: almost all)
REPEAT = 5 # jumps back of the same target, each within a day of the other, that make it a flipping target (PR_BG of
# CLARIDEN, batch 01: about thirty in 2.5 days)
def _jumps(t, v):
"""A series (instants, values) -> [(start i, end i, state)], state 'back', 'lasting' or 'recent'."""
day = timedelta(hours=BACK_H)
noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:]))
step = statistics.median(b - a for a, b in zip(t, t[1:])) # cadence of the series
back_n = max(1, min(BASE, round(timedelta(hours=1) / step))) # measurements of an hour (BASE at 15 min, one at 4 h)
# ponytail: usual place = first day of the export, wrong if it starts within an artefact
ref = statistics.median(x for x, u in zip(v, t) if u < t[0] + day)
lv = {} # level of each UTC day; its ordinary variation from one day to the next judges a jump under K
for x, u in zip(v, t):
lv.setdefault(u.date(), []).append(x)
lv = [statistics.median(x) for _, x in sorted(lv.items())]
ordinary = statistics.median(abs(b - a) for a, b in zip(lv, lv[1:])) if len(lv) > 2 else 0
# the first BASE measurements, lacking a level before, face the level after: a gap there came back at once
out = [(k, k, "back") for k in range(min(BASE, len(v) - BASE))
if noise and abs(v[k] - statistics.median(v[k + 1:k + 1 + BASE])) >= K * noise]
i = BASE
while noise and i < len(v):
base = statistics.median(v[i - BASE:i])
if abs(v[i] - base) < K_LASTING * noise:
i += 1
continue
tol = abs(v[i] - base) / 2 # back: an hour of measurements in a row within half the jump from the level before
j = next((k for k in range(i + 1, len(v)) if all(abs(x - base) < tol for x in v[k:k + back_n])), len(v))
before = statistics.median([x for x, u in zip(v, t) if t[i] - day <= u < t[i]] or [base])
after = statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + day)
if j < len(v) and t[j] - t[i] <= day:
state = "back"
elif abs(after - ref) < tol or abs(after - before) < tol: # the day after, the target is at its usual place,
state, j = "back", i + 1 # or where it was the day before: the jump ended a dip
else:
state = "lasting" if t[-1] - t[i] >= day else "recent" # less than a day of hindsight: not known yet
if abs(v[i] - base) < K * noise and (state != "lasting" or (after - before) * (v[i] - base) < 0
or not ordinary or abs(after - before) < PLATEAU * ordinary):
i += 1 # under K, only a jump that stays counts: the day after far from the day before, on its side
continue
out.append((i, j - 1, state))
# level before rebuilt after the return; never back: the new level becomes the level before, the rest is
# still read (PR_BG of CLARIDEN, batch 01: its flips after the 10:30 jump were invisible)
i = (j if j < len(v) else i) + BASE
return out
def _peaks(t, v):
"""Instants where the measurement moves away by at least K_PEAK noises from both neighbours, same side (one-step peak)."""
noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:]))
return [t[i] for i in range(1, len(v) - 1) if noise and (v[i] - v[i - 1]) * (v[i] - v[i + 1]) > 0
and min(abs(v[i] - v[i - 1]), abs(v[i] - v[i + 1])) >= K_PEAK * noise]
def episodes(measurements):
"""{sensor: [measurement]} (data.load) -> jump episodes, sorted by start: Displacement, and Target issue for a
flipping target."""
found = [] # (start, end, state, sensor)
peaks = {} # instant -> targets peaking there for one step
for name, ms in measurements.items():
if data.is_reference(ms):
continue
for axis in ("corr_x", "corr_y"):
kept = [m for m in ms if m[axis] is not None]
if len(kept) <= BASE:
continue
t, v = [m["t"] for m in kept], [m[axis] for m in kept]
# a jump that did not come back ends with itself, not with the data (its bar hid the others)
found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)]
for u in _peaks(t, v):
peaks.setdefault(u, set()).add(name)
found += [(u, u, "back", s) for u, cs in peaks.items() if len(cs) >= PEAK_MIN for s in cs] # common peak: back
alone = _flips(found) # a target that keeps jumping and coming back: an episode on its own, outside the common ones
found = [f for f in found if f[2] != "back" or not any(f[3] in g["sensors"] and g["start"] <= f[0] <= g["end"]
for g in alone)]
groups = [] # a jump that did not come back only counts by its instant: its end is often the end of the data
for s, e, state, name in sorted(found):
if groups and s <= groups[-1]["until"] + GAP:
g = groups[-1]
else:
g = {"start": s, "end": e, "until": s, "last": s, "sensors": {}}
groups.append(g)
g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s
g["sensors"].setdefault(name, set()).add(state)
long = [g for g in alone if g["flip"]] # flips of many targets at the same time: the compensation, not a
for g in long: # problem target
if sum(h["start"] <= g["end"] and h["end"] >= g["start"] for h in long) > MAX_FLIPPING:
g["flip"] = 0
total = sum(not data.is_reference(ms) for ms in measurements.values())
return sorted((_episode(g, total) for g in groups + alone), key=lambda ep: ep["start"])
def _flips(found):
"""Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> a group, from the first
to the last flip."""
day, alone = timedelta(hours=BACK_H), []
for name in {f[3] for f in found}:
chain = [] # (start, end); corr_x and corr_y twice: counted by instant
for s, e in sorted({(s, e) for s, e, state, n in found if n == name and state == "back"}) + [(None, None)]:
if chain and s is not None and s <= max(c[1] for c in chain) + day:
chain.append((s, e))
continue
if len({c[0] for c in chain}) >= REPEAT:
alone.append({"start": chain[0][0], "end": max(c[1] for c in chain), "sensors": {name: {"back"}},
"flip": len({c[0] for c in chain}) if max(c[1] for c in chain) - chain[0][0] >= FLIP else 0})
chain = [(s, e)]
return alone
def _episode(g, total):
"""A group -> episode; a single lasting (or too recent) jump is enough not to grey; a flipping target: "Target
flipping" (Target issue, D-074); a lasting jump of LARGE targets or more: "Large shift"."""
n = len(g["sensors"])
lasting = sum("lasting" in st for st in g["sensors"].values())
recent = any("recent" in st for st in g["sensors"].values())
who = "One target" if n == 1 else f"{n} targets together (a single event)"
if g.get("flip"):
name = next(iter(g["sensors"]))
return {"start": data.iso(g["start"]), "end": data.iso(g["end"]), "kind": "Target issue", "label": "Target flipping",
"text": f"{name} jumped and came back {g['flip']} times: its tracking is not reliable, look at the images "
"of the target.", "ignore": True, "sensors": [name], "lasting": []}
if lasting and n >= LARGE * total:
label = "Large shift"
text = f"{n} targets jumped together and {lasting} of them stayed shifted: everything moved, please check it."
elif lasting:
label = "Lasting shift"
some = "" if lasting == n else f", {lasting} of them stayed shifted for more than a day"
text = (f"{who} jumped{some}" if some else f"{who} jumped and stayed shifted for more than a day") + \
": this may be a real movement, please check it."
elif recent:
label = "Jump, not back yet"
text = f"{who} jumped less than a day before the last measurement and has not come back yet: check again later."
else:
label = "Sudden jump, back to normal"
text = f"{who} jumped and came back within a day: the cause is not established, look at the images."
end = g["last"] if lasting or recent else g["end"] # not greyed: the last jump, not the returns
return {"start": data.iso(g["start"]), "end": data.iso(end), "kind": "Displacement", "label": label,
"text": text, "ignore": not lasting and not recent, "sensors": sorted(g["sensors"]),
"lasting": sorted(s for s, st in g["sensors"].items() if "lasting" in st)} # stayed shifted (event sentence)
def detect(measurements):
return sorted([e for e in episodes(measurements) if e["kind"] == "Displacement"] + drift(measurements),
key=lambda ep: ep["start"])
# --- Slow drift ------------------------------------------------------------------------------------------------------
# A target that moves little by little, day after day. Takes up the trend of the Displacements tab (D-055) on the
# "advanced" export (corr_x, corr_y, px). Method, per target (global references excluded) and per axis:
# 1. Level of the day: median of the measurements of each UTC day with at least HOURS measured hours out of 24, as a
# share of the hours the station measures (wf-improve MICHIGAN, mod. 1: at one measurement every 4 h, 5 of the 6 hours;
# before, no day counted); at least DAYS days.
# 2. Theil-Sen slope and Kendall tau of the daily levels; amplitude of the day: gap between its 9th and 1st deciles.
# 3. Drift: |slope| × duration ≥ SIZE times the median daily amplitude, |tau| ≥ STEADY, and no step from one day to the
# next of more than half the drift (a jump that lasts is not a drift: the jump rule takes it).
# 4. One episode per target (one axis is enough), from the first measurement of the first day to the last of the last
# day; not greyed.
# 4b. Piecewise (wf-improve session Displacement, mod. 1, S1): a target without a drift over the whole window may drift for
# a few days, then settle or go back. On each run of at least DAYS kept days, drift = level of the last day minus
# that of the first, at least PIECE daily amplitudes (median of the series), |tau| ≥ STEADY with the same sign, no
# step from one kept day to the next of more than half the drift; pieces taken from the largest drift to the
# smallest, without overlapping (they may share the day where the level turns): each goes from one extreme of the
# level to the other, one episode per piece and per axis. At PIECE = 1, the thermal cycles of a few days and the
# noise of the levels get through (JUNGFRAU, NORDEND): 3.5 keeps the rows of CLARIDEN (3.8 to 8.3 amplitudes),
# drops JUNGFRAU (3.1 at most). The pieces do not replace step 3. A piece does not cross a hole: two consecutive kept
# days are at most MAX_HOLE days apart (mod. 2: the gap of a level across three days without a measurement is a
# step, not a slope; it made pieces of the whole window, CLARIDEN 09-28 → 11-01). A missing day does not cut.
# 5. Camera drift (same mod.): at least LARGE of the targets drift on the same side on corr_x or corr_y, and at least
# LARGE of the sensors, references included, on their raw positions x or y: only the pieces of point 4b are
# reported, not the trend over the whole window (mod. 1, S2: the badly compensated drifting camera, NEBRASKA, is a
# Displacement for the labels). The targets of a bridge following the season (JUNGFRAU, labelled Thermal
# displacement day after day) pass step 3 but not PIECE; NEBRASKA both.
HOURS = 20 # a day counts if it has at least 20 measured hours out of 24 (as D-055), as a share of the measured hours
DAYS = 5 # minimum days for a trend (D-055)
SIZE = 1.0 # drift ≥ SIZE daily amplitudes (D-055)
STEADY = 0.6 # |Kendall tau| of the daily levels (D-055)
PIECE = 3.5 # piecewise drift ≥ PIECE daily amplitudes (point 4b)
MAX_HOLE = 2 # a piece does not go from one kept day to the next more than MAX_HOLE days apart (point 4b)
def _decile(vs, q):
vs = sorted(vs)
return vs[min(len(vs) - 1, int(q * len(vs)))]
def _days(ms, col):
"""-> {UTC day: its measurements}, the kept days (point 1)."""
days = {}
for m in ms:
if m[col] is not None:
days.setdefault(m["t"].date(), []).append(m)
slots = len({m["t"].hour for dm in days.values() for m in dm}) # hours the station measures (6 at a 4 h step)
return {d: dm for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots}
def _amplitude(days, col):
return statistics.median(_decile([m[col] for m in dm], 0.9) - _decile([m[col] for m in dm], 0.1) for dm in days.values())
def _drift(ms, col):
"""-> (drift in px, days, first instant, last) or None."""
days = _days(ms, col)
if len(days) < DAYS:
return None
lv = sorted((d.toordinal(), statistics.median(m[col] for m in dm)) for d, dm in days.items())
pairs = [(a, b) for k, a in enumerate(lv) for b in lv[k + 1:]]
slope = statistics.median((b[1] - a[1]) / (b[0] - a[0]) for a, b in pairs)
tau = sum((b[1] > a[1]) - (b[1] < a[1]) for a, b in pairs) / len(pairs)
drift = slope * (lv[-1][0] - lv[0][0])
step = max(abs(b[1] - a[1]) for a, b in zip(lv, lv[1:]))
if abs(drift) < SIZE * _amplitude(days, col) or abs(tau) < STEADY or step > abs(drift) / 2:
return None
d0, d1 = min(days), max(days)
return drift, len(days), min(m["t"] for m in days[d0]), max(m["t"] for m in days[d1])
def _pieces(ms, col):
"""-> [(drift in px, days, first instant, last)]: the drift pieces (point 4b)."""
days = _days(ms, col)
if len(days) < DAYS:
return []
ds = sorted(days)
lv = [statistics.median(m[col] for m in days[d]) for d in ds]
ampl = _amplitude(days, col)
pieces = [] # (drift, i, j)
for i in range(len(ds)):
signs = step = 0
for j in range(i + 1, len(ds)):
if (ds[j] - ds[j - 1]).days > MAX_HOLE:
break # a gap across a hole is a step (the jump rule takes it), not a slope
signs += sum((lv[j] > lv[k]) - (lv[j] < lv[k]) for k in range(i, j)) # Kendall tau, step by step
step = max(step, abs(lv[j] - lv[j - 1]))
n, d = j - i + 1, lv[j] - lv[i]
if n >= DAYS and d and abs(d) >= PIECE * ampl and step <= abs(d) / 2 \
and signs * (1 if d > 0 else -1) >= STEADY * n * (n - 1) / 2:
pieces.append((d, i, j))
taken = []
for d, i, j in sorted(pieces, key=lambda x: (-abs(x[0]), x[2] - x[1])):
if all(j <= a or i >= b for _, a, b in taken):
taken.append((d, i, j))
return [(d, j - i + 1, min(m["t"] for m in days[ds[i]]), max(m["t"] for m in days[ds[j]])) for d, i, j in taken]
def _common(measurements, cols):
"""At least LARGE of the sensors drift on the same side on the same axis of cols."""
for col in cols:
signs = [x[0] > 0 for ms in measurements.values() if (x := _drift(ms, col))]
if max(signs.count(True), signs.count(False)) >= LARGE * len(measurements):
return True
return False
def drift(measurements):
targets = {s: ms for s, ms in measurements.items() if not data.is_reference(ms)}
camera = _common(targets, ("corr_x", "corr_y")) and _common(measurements, ("x", "y")) # point 5: pieces only
out = []
for s, ms in sorted(targets.items()):
r = [] if camera else [x for col in ("corr_x", "corr_y") if (x := _drift(ms, col))]
r = [max(r, key=lambda x: abs(x[0]))] if r else [x for col in ("corr_x", "corr_y") for x in _pieces(ms, col)]
out += [{"start": data.iso(a), "end": data.iso(b), "kind": "Displacement", "label": "Slow drift",
"text": f"{s} drifts slowly and steadily: {d:+.2f} px in {n} days. Keep watching it.",
"ignore": False, "sensors": [s]} for d, n, a, b in r]
return out
if __name__ == "__main__":
for ep in detect(data.load(sys.argv[1])):
print(ep["start"], ep["end"], "ignore" if ep["ignore"] else "KEEP ", ep["label"], "|", len(ep["sensors"]),
",".join(ep["sensors"][:6]), "|", ep["text"])