Files
aimsight-detectors/scripts/displacement.py
T

190 lines
9.9 KiB
Python
Raw Normal View History

"""Displacement family: sudden jumps of the corrected displacements (corr_x, corr_y, px) of each target, and slow drift
(end of file). Global references are excluded, rejected measurements (empty corr) skipped.
1. Jump: a measurement at least K noises (median step of the series) away from the level before (median of the BASE
previous measurements).
2. Back to normal: within BACK_H, an hour of measurements in a row (BASE at most, one at least) within half the jump of
the level before; or the day after the jump at the level of the day before (the jump ended a dip). Otherwise
"Lasting shift", or "Jump, not back yet" with less than BACK_H of hindsight. After a jump that never comes back, its
new level becomes the level before.
3. Jumps of all the targets less than GAP apart make one episode; a target that jumps and comes back at least REPEAT
times, each within BACK_H of the previous one, makes an episode of its own, from its first jump to its last return.
4. "Sudden jump, back to normal" is greyed and its cause is not established here: categories.py gives it the family of
an overlapping episode (rain, something passing...), otherwise it is to be checked on the images. A shift is never
greyed (maybe a real movement); its episode ends at its last jump.
"""
import statistics
from datetime import timedelta
from common import episode, noise, targets
from data import is_reference
K = 20 # E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 are visibility's
BASE = 4 # measurements of the level before (1 h at 15 min); the return is judged on an hour (wf-improve MICHIGAN, mod. 1)
GAP = timedelta(hours=1)
BACK_H = timedelta(hours=24) # E-002: plateaus up to 11.5 h
REPEAT = 5 # CLARIDEN, batch 01: PR_BG flips 28 times in 2.5 days, the other targets 3 at most
JUMP_BACK = "Sudden jump, back to normal"
def _jumps(t, v):
"""A series (instants, values) -> [(start index, end index, state)], state "back", "lasting" or "recent"."""
n = noise(v)
back_n = max(1, min(BASE, round(timedelta(hours=1) / statistics.median(b - a for a, b in zip(t, t[1:])))))
out, i = [], BASE
while n and i < len(v):
base = statistics.median(v[i - BASE:i])
if abs(v[i] - base) < K * n:
i += 1
continue
tol = abs(v[i] - base) / 2
j = next((k for k in range(i + 1, len(v)) if all(abs(x - base) < tol for x in v[k:k + back_n])), len(v))
if j < len(v) and t[j] - t[i] <= BACK_H:
state = "back"
elif abs(statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + BACK_H)
- statistics.median([x for x, u in zip(v, t) if t[i] - BACK_H <= u < t[i]] or [base])) < tol:
state, j = "back", i + 1 # the day after at the level of the day before: the jump ended a dip
else:
state = "lasting" if t[-1] - t[i] >= BACK_H else "recent"
out.append((i, j - 1, state))
i = (j if j < len(v) else i) + BASE
return out
def _repeats(found):
"""Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> [(start, end, target)]."""
out = []
for name in {f[3] for f in found}:
chain = [] # (start, end); corr_x and corr_y together: counted by instant
for s, e in sorted({(s, e) for s, e, state, n in found if n == name and state == "back"}) + [(None, None)]:
if chain and s is not None and s <= max(c[1] for c in chain) + BACK_H:
chain.append((s, e))
continue
if len({c[0] for c in chain}) >= REPEAT:
out.append((chain[0][0], max(c[1] for c in chain), name))
chain = [(s, e)]
return out
def _episode(start, end, states):
"""{target: {states}} -> the episode of a group of jumps."""
n = len(states)
lasting = sum("lasting" in st for st in states.values())
who = targets(n).capitalize()
if lasting:
some = "stayed shifted" if lasting == n else f"{lasting} of them stayed shifted"
label, text = "Lasting shift", f"{who} shifted; {some}."
elif any("recent" in st for st in states.values()):
label, text = "Jump, not back yet", f"{who} shifted and {'has' if n == 1 else 'have'} not come back yet."
else:
return episode(start, end, "Displacement", JUMP_BACK, f"{who} jumped and came back. The cause is not established: "
"look at the images.", states, True)
return episode(start, end, "Displacement", label, text + " Check whether the structure moved: look at the images.",
states, False)
def jumps(measurements):
found = [] # (start, end, state, target)
for name, ms in measurements.items():
if is_reference(ms):
continue
for axis in ("corr_x", "corr_y"):
kept = [m for m in ms if m[axis] is not None]
if len(kept) > BASE:
t, v = [m["t"] for m in kept], [m[axis] for m in kept]
found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)]
alone = _repeats(found)
out = [_episode(a, b, {name: {"back"}}) for a, b, name in alone]
groups = [] # a shift only counts by its instant: its end is often the end of the data
for s, e, state, name in sorted(found):
if state == "back" and any(name == n and a <= s <= b for a, b, n in alone):
continue
if not groups or s > groups[-1]["until"] + GAP:
groups.append({"start": s, "end": e, "until": s, "states": {}})
g = groups[-1]
g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s
g["states"].setdefault(name, set()).add(state)
for g in groups:
shift = any(st != {"back"} for st in g["states"].values())
out.append(_episode(g["start"], g["last"] if shift else g["end"], g["states"]))
return out
def detect(measurements):
return jumps(measurements) + drift(measurements)
# --- Slow drift: a target that moves little by little, day after day (the trend of the Displacements tab, D-055).
# Per target and per axis (corr_x, corr_y): the level of each UTC day measured at least HOURS of the hours the station
# measures (median of the day), at least DAYS days; the amplitude of a day: its 9th minus its 1st decile.
# 1. Over the whole window: Theil-Sen slope × duration at least SIZE median daily amplitudes, |Kendall tau| ≥ STEADY, and
# no step from one day to the next larger than half the drift (that is a jump).
# 2. Otherwise, pieces: runs of at least DAYS kept days whose last level minus the first is at least PIECE amplitudes, with
# the same tau and step conditions; taken from the largest to the smallest without overlapping (wf-improve
# Displacement, mod. 1, S1: rows of CLARIDEN, 3.8 to 8.3 amplitudes; thermal cycles of JUNGFRAU, 3.1 at most).
# One episode per drift, not greyed: the largest of the two axes over the whole window, otherwise each piece.
HOURS = 20 # out of 24, as a share of the hours the station measures (wf-improve MICHIGAN, mod. 1)
DAYS = 5 # D-055
SIZE = 1.0 # D-055
STEADY = 0.6 # D-055
PIECE = 3.5 # wf-improve Displacement, mod. 1, S1
def _decile(vs, q):
vs = sorted(vs)
return vs[min(len(vs) - 1, int(q * len(vs)))]
def _days(ms, col):
"""-> (kept days, their levels, median daily amplitude, [(first instant, last) of each day])."""
days = {}
for m in ms:
if m[col] is not None:
days.setdefault(m["t"].date(), []).append(m)
slots = len({m["t"].hour for dm in days.values() for m in dm}) # hours the station measures (6 at a 4 h step)
ds = sorted(d for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots)
vals = [[m[col] for m in days[d]] for d in ds]
ampl = statistics.median(_decile(v, 0.9) - _decile(v, 0.1) for v in vals) if ds else 0
return ds, [statistics.median(v) for v in vals], ampl, [(days[d][0]["t"], days[d][-1]["t"]) for d in ds]
def _whole(ds, lv, ampl, _):
"""-> [(drift, 0, last index)] if the levels drift over the whole window, else []."""
pairs = [(i, j) for i in range(len(ds)) for j in range(i + 1, len(ds))]
slope = statistics.median((lv[j] - lv[i]) / (ds[j] - ds[i]).days for i, j in pairs)
tau = sum((lv[j] > lv[i]) - (lv[j] < lv[i]) for i, j in pairs) / len(pairs)
d = slope * (ds[-1] - ds[0]).days
step = max(abs(b - a) for a, b in zip(lv, lv[1:]))
return [(d, 0, len(ds) - 1)] if abs(d) >= SIZE * ampl and abs(tau) >= STEADY and step <= abs(d) / 2 else []
def _pieces(ds, lv, ampl, _):
"""-> [(drift, first index, last index)]: the drift pieces, without overlapping."""
pieces = []
for i in range(len(ds)):
signs = step = 0
for j in range(i + 1, len(ds)):
signs += sum((lv[j] > lv[k]) - (lv[j] < lv[k]) for k in range(i, j)) # Kendall tau, step by step
step = max(step, abs(lv[j] - lv[j - 1]))
n, d = j - i + 1, lv[j] - lv[i]
if n >= DAYS and d and abs(d) >= PIECE * ampl and step <= abs(d) / 2 \
and signs * (1 if d > 0 else -1) >= STEADY * n * (n - 1) / 2:
pieces.append((d, i, j))
taken = []
for d, i, j in sorted(pieces, key=lambda x: (-abs(x[0]), x[2] - x[1])):
if all(j <= a or i >= b for _, a, b in taken):
taken.append((d, i, j))
return taken
def drift(measurements):
out = []
for s, ms in sorted(measurements.items()):
if is_reference(ms):
continue
axes = [a for col in ("corr_x", "corr_y") if len((a := _days(ms, col))[0]) >= DAYS]
whole = [(w, a) for a in axes for w in _whole(*a)]
found = [max(whole, key=lambda x: abs(x[0][0]))] if whole else [(p, a) for a in axes for p in _pieces(*a)]
out += [episode(a[3][i][0], a[3][j][1], "Displacement", "Slow drift", f"{s} drifts slowly and steadily: {d:+.2f} "
f"px in {j - i + 1} days. Keep watching it.", [s], False) for (d, i, j), a in found]
return out