2026-11-16 17:20:17 +01:00
|
|
|
|
"""Displacement family: sudden jumps of the corrected displacements (corr_x, corr_y, px) of each target, and slow drift
|
|
|
|
|
|
(end of file). Global references are excluded, rejected measurements (empty corr) skipped.
|
|
|
|
|
|
|
|
|
|
|
|
1. Jump: a measurement at least K noises (median step of the series) away from the level before (median of the BASE
|
|
|
|
|
|
previous measurements).
|
|
|
|
|
|
2. Back to normal: within BACK_H, an hour of measurements in a row (BASE at most, one at least) within half the jump of
|
|
|
|
|
|
the level before; or the day after the jump at the level of the day before (the jump ended a dip). Otherwise
|
|
|
|
|
|
"Lasting shift", or "Jump, not back yet" with less than BACK_H of hindsight. After a jump that never comes back, its
|
|
|
|
|
|
new level becomes the level before.
|
|
|
|
|
|
3. Jumps of all the targets less than GAP apart make one episode; a target that jumps and comes back at least REPEAT
|
|
|
|
|
|
times, each within BACK_H of the previous one, makes an episode of its own, from its first jump to its last return.
|
|
|
|
|
|
4. "Sudden jump, back to normal" is greyed and its cause is not established here: categories.py gives it the family of
|
|
|
|
|
|
an overlapping episode (rain, something passing...), otherwise it is to be checked on the images. A shift is never
|
|
|
|
|
|
greyed (maybe a real movement); its episode ends at its last jump.
|
2026-11-13 12:19:52 +01:00
|
|
|
|
"""
|
|
|
|
|
|
import statistics
|
|
|
|
|
|
from datetime import timedelta
|
|
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
from common import episode, noise, targets
|
|
|
|
|
|
from data import is_reference
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
K = 20 # E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 are visibility's
|
|
|
|
|
|
BASE = 4 # measurements of the level before (1 h at 15 min); the return is judged on an hour (wf-improve MICHIGAN, mod. 1)
|
|
|
|
|
|
GAP = timedelta(hours=1)
|
|
|
|
|
|
BACK_H = timedelta(hours=24) # E-002: plateaus up to 11.5 h
|
|
|
|
|
|
REPEAT = 5 # CLARIDEN, batch 01: PR_BG flips 28 times in 2.5 days, the other targets 3 at most
|
|
|
|
|
|
JUMP_BACK = "Sudden jump, back to normal"
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _jumps(t, v):
|
2026-11-16 17:20:17 +01:00
|
|
|
|
"""A series (instants, values) -> [(start index, end index, state)], state "back", "lasting" or "recent"."""
|
|
|
|
|
|
n = noise(v)
|
|
|
|
|
|
back_n = max(1, min(BASE, round(timedelta(hours=1) / statistics.median(b - a for a, b in zip(t, t[1:])))))
|
|
|
|
|
|
out, i = [], BASE
|
|
|
|
|
|
while n and i < len(v):
|
2026-11-13 12:19:52 +01:00
|
|
|
|
base = statistics.median(v[i - BASE:i])
|
2026-11-16 17:20:17 +01:00
|
|
|
|
if abs(v[i] - base) < K * n:
|
2026-11-13 12:19:52 +01:00
|
|
|
|
i += 1
|
|
|
|
|
|
continue
|
2026-11-16 17:20:17 +01:00
|
|
|
|
tol = abs(v[i] - base) / 2
|
2026-11-13 12:19:52 +01:00
|
|
|
|
j = next((k for k in range(i + 1, len(v)) if all(abs(x - base) < tol for x in v[k:k + back_n])), len(v))
|
2026-11-16 17:20:17 +01:00
|
|
|
|
if j < len(v) and t[j] - t[i] <= BACK_H:
|
2026-11-13 12:19:52 +01:00
|
|
|
|
state = "back"
|
2026-11-16 17:20:17 +01:00
|
|
|
|
elif abs(statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + BACK_H)
|
|
|
|
|
|
- statistics.median([x for x, u in zip(v, t) if t[i] - BACK_H <= u < t[i]] or [base])) < tol:
|
|
|
|
|
|
state, j = "back", i + 1 # the day after at the level of the day before: the jump ended a dip
|
2026-11-13 12:19:52 +01:00
|
|
|
|
else:
|
2026-11-16 17:20:17 +01:00
|
|
|
|
state = "lasting" if t[-1] - t[i] >= BACK_H else "recent"
|
2026-11-13 12:19:52 +01:00
|
|
|
|
out.append((i, j - 1, state))
|
|
|
|
|
|
i = (j if j < len(v) else i) + BASE
|
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
def _repeats(found):
|
|
|
|
|
|
"""Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> [(start, end, target)]."""
|
|
|
|
|
|
out = []
|
2026-11-13 12:19:52 +01:00
|
|
|
|
for name in {f[3] for f in found}:
|
2026-11-16 17:20:17 +01:00
|
|
|
|
chain = [] # (start, end); corr_x and corr_y together: counted by instant
|
2026-11-13 12:19:52 +01:00
|
|
|
|
for s, e in sorted({(s, e) for s, e, state, n in found if n == name and state == "back"}) + [(None, None)]:
|
2026-11-16 17:20:17 +01:00
|
|
|
|
if chain and s is not None and s <= max(c[1] for c in chain) + BACK_H:
|
2026-11-13 12:19:52 +01:00
|
|
|
|
chain.append((s, e))
|
|
|
|
|
|
continue
|
|
|
|
|
|
if len({c[0] for c in chain}) >= REPEAT:
|
2026-11-16 17:20:17 +01:00
|
|
|
|
out.append((chain[0][0], max(c[1] for c in chain), name))
|
2026-11-13 12:19:52 +01:00
|
|
|
|
chain = [(s, e)]
|
2026-11-16 17:20:17 +01:00
|
|
|
|
return out
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
def _episode(start, end, states):
|
|
|
|
|
|
"""{target: {states}} -> the episode of a group of jumps."""
|
|
|
|
|
|
n = len(states)
|
|
|
|
|
|
lasting = sum("lasting" in st for st in states.values())
|
|
|
|
|
|
who = targets(n).capitalize()
|
|
|
|
|
|
if lasting:
|
|
|
|
|
|
some = "stayed shifted" if lasting == n else f"{lasting} of them stayed shifted"
|
|
|
|
|
|
label, text = "Lasting shift", f"{who} shifted; {some}."
|
|
|
|
|
|
elif any("recent" in st for st in states.values()):
|
|
|
|
|
|
label, text = "Jump, not back yet", f"{who} shifted and {'has' if n == 1 else 'have'} not come back yet."
|
2026-11-13 12:19:52 +01:00
|
|
|
|
else:
|
2026-11-16 17:20:17 +01:00
|
|
|
|
return episode(start, end, "Displacement", JUMP_BACK, f"{who} jumped and came back. The cause is not established: "
|
|
|
|
|
|
"look at the images.", states, True)
|
|
|
|
|
|
return episode(start, end, "Displacement", label, text + " Check whether the structure moved: look at the images.",
|
|
|
|
|
|
states, False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def jumps(measurements):
|
|
|
|
|
|
found = [] # (start, end, state, target)
|
|
|
|
|
|
for name, ms in measurements.items():
|
|
|
|
|
|
if is_reference(ms):
|
|
|
|
|
|
continue
|
|
|
|
|
|
for axis in ("corr_x", "corr_y"):
|
|
|
|
|
|
kept = [m for m in ms if m[axis] is not None]
|
|
|
|
|
|
if len(kept) > BASE:
|
|
|
|
|
|
t, v = [m["t"] for m in kept], [m[axis] for m in kept]
|
|
|
|
|
|
found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)]
|
|
|
|
|
|
alone = _repeats(found)
|
|
|
|
|
|
out = [_episode(a, b, {name: {"back"}}) for a, b, name in alone]
|
|
|
|
|
|
groups = [] # a shift only counts by its instant: its end is often the end of the data
|
|
|
|
|
|
for s, e, state, name in sorted(found):
|
|
|
|
|
|
if state == "back" and any(name == n and a <= s <= b for a, b, n in alone):
|
|
|
|
|
|
continue
|
|
|
|
|
|
if not groups or s > groups[-1]["until"] + GAP:
|
|
|
|
|
|
groups.append({"start": s, "end": e, "until": s, "states": {}})
|
|
|
|
|
|
g = groups[-1]
|
|
|
|
|
|
g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s
|
|
|
|
|
|
g["states"].setdefault(name, set()).add(state)
|
|
|
|
|
|
for g in groups:
|
|
|
|
|
|
shift = any(st != {"back"} for st in g["states"].values())
|
|
|
|
|
|
out.append(_episode(g["start"], g["last"] if shift else g["end"], g["states"]))
|
|
|
|
|
|
return out
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def detect(measurements):
|
2026-11-16 17:20:17 +01:00
|
|
|
|
return jumps(measurements) + drift(measurements)
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
# --- Slow drift: a target that moves little by little, day after day (the trend of the Displacements tab, D-055).
|
|
|
|
|
|
# Per target and per axis (corr_x, corr_y): the level of each UTC day measured at least HOURS of the hours the station
|
|
|
|
|
|
# measures (median of the day), at least DAYS days; the amplitude of a day: its 9th minus its 1st decile.
|
|
|
|
|
|
# 1. Over the whole window: Theil-Sen slope × duration at least SIZE median daily amplitudes, |Kendall tau| ≥ STEADY, and
|
|
|
|
|
|
# no step from one day to the next larger than half the drift (that is a jump).
|
|
|
|
|
|
# 2. Otherwise, pieces: runs of at least DAYS kept days whose last level minus the first is at least PIECE amplitudes, with
|
|
|
|
|
|
# the same tau and step conditions; taken from the largest to the smallest without overlapping (wf-improve
|
|
|
|
|
|
# Displacement, mod. 1, S1: rows of CLARIDEN, 3.8 to 8.3 amplitudes; thermal cycles of JUNGFRAU, 3.1 at most).
|
|
|
|
|
|
# One episode per drift, not greyed: the largest of the two axes over the whole window, otherwise each piece.
|
|
|
|
|
|
HOURS = 20 # out of 24, as a share of the hours the station measures (wf-improve MICHIGAN, mod. 1)
|
|
|
|
|
|
DAYS = 5 # D-055
|
|
|
|
|
|
SIZE = 1.0 # D-055
|
|
|
|
|
|
STEADY = 0.6 # D-055
|
|
|
|
|
|
PIECE = 3.5 # wf-improve Displacement, mod. 1, S1
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _decile(vs, q):
|
|
|
|
|
|
vs = sorted(vs)
|
|
|
|
|
|
return vs[min(len(vs) - 1, int(q * len(vs)))]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _days(ms, col):
|
2026-11-16 17:20:17 +01:00
|
|
|
|
"""-> (kept days, their levels, median daily amplitude, [(first instant, last) of each day])."""
|
2026-11-13 12:19:52 +01:00
|
|
|
|
days = {}
|
|
|
|
|
|
for m in ms:
|
|
|
|
|
|
if m[col] is not None:
|
|
|
|
|
|
days.setdefault(m["t"].date(), []).append(m)
|
|
|
|
|
|
slots = len({m["t"].hour for dm in days.values() for m in dm}) # hours the station measures (6 at a 4 h step)
|
2026-11-16 17:20:17 +01:00
|
|
|
|
ds = sorted(d for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots)
|
|
|
|
|
|
vals = [[m[col] for m in days[d]] for d in ds]
|
|
|
|
|
|
ampl = statistics.median(_decile(v, 0.9) - _decile(v, 0.1) for v in vals) if ds else 0
|
|
|
|
|
|
return ds, [statistics.median(v) for v in vals], ampl, [(days[d][0]["t"], days[d][-1]["t"]) for d in ds]
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
def _whole(ds, lv, ampl, _):
|
|
|
|
|
|
"""-> [(drift, 0, last index)] if the levels drift over the whole window, else []."""
|
|
|
|
|
|
pairs = [(i, j) for i in range(len(ds)) for j in range(i + 1, len(ds))]
|
|
|
|
|
|
slope = statistics.median((lv[j] - lv[i]) / (ds[j] - ds[i]).days for i, j in pairs)
|
|
|
|
|
|
tau = sum((lv[j] > lv[i]) - (lv[j] < lv[i]) for i, j in pairs) / len(pairs)
|
|
|
|
|
|
d = slope * (ds[-1] - ds[0]).days
|
|
|
|
|
|
step = max(abs(b - a) for a, b in zip(lv, lv[1:]))
|
|
|
|
|
|
return [(d, 0, len(ds) - 1)] if abs(d) >= SIZE * ampl and abs(tau) >= STEADY and step <= abs(d) / 2 else []
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 17:20:17 +01:00
|
|
|
|
def _pieces(ds, lv, ampl, _):
|
|
|
|
|
|
"""-> [(drift, first index, last index)]: the drift pieces, without overlapping."""
|
|
|
|
|
|
pieces = []
|
2026-11-13 12:19:52 +01:00
|
|
|
|
for i in range(len(ds)):
|
|
|
|
|
|
signs = step = 0
|
|
|
|
|
|
for j in range(i + 1, len(ds)):
|
|
|
|
|
|
signs += sum((lv[j] > lv[k]) - (lv[j] < lv[k]) for k in range(i, j)) # Kendall tau, step by step
|
|
|
|
|
|
step = max(step, abs(lv[j] - lv[j - 1]))
|
|
|
|
|
|
n, d = j - i + 1, lv[j] - lv[i]
|
|
|
|
|
|
if n >= DAYS and d and abs(d) >= PIECE * ampl and step <= abs(d) / 2 \
|
|
|
|
|
|
and signs * (1 if d > 0 else -1) >= STEADY * n * (n - 1) / 2:
|
|
|
|
|
|
pieces.append((d, i, j))
|
|
|
|
|
|
taken = []
|
|
|
|
|
|
for d, i, j in sorted(pieces, key=lambda x: (-abs(x[0]), x[2] - x[1])):
|
|
|
|
|
|
if all(j <= a or i >= b for _, a, b in taken):
|
|
|
|
|
|
taken.append((d, i, j))
|
2026-11-16 17:20:17 +01:00
|
|
|
|
return taken
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def drift(measurements):
|
|
|
|
|
|
out = []
|
2026-11-16 17:20:17 +01:00
|
|
|
|
for s, ms in sorted(measurements.items()):
|
|
|
|
|
|
if is_reference(ms):
|
|
|
|
|
|
continue
|
|
|
|
|
|
axes = [a for col in ("corr_x", "corr_y") if len((a := _days(ms, col))[0]) >= DAYS]
|
|
|
|
|
|
whole = [(w, a) for a in axes for w in _whole(*a)]
|
|
|
|
|
|
found = [max(whole, key=lambda x: abs(x[0][0]))] if whole else [(p, a) for a in axes for p in _pieces(*a)]
|
|
|
|
|
|
out += [episode(a[3][i][0], a[3][j][1], "Displacement", "Slow drift", f"{s} drifts slowly and steadily: {d:+.2f} "
|
|
|
|
|
|
f"px in {j - i + 1} days. Keep watching it.", [s], False) for (d, i, j), a in found]
|
2026-11-13 12:19:52 +01:00
|
|
|
|
return out
|