diff --git a/scripts/displacement.py b/scripts/displacement.py index fc92e80..16cbb50 100644 --- a/scripts/displacement.py +++ b/scripts/displacement.py @@ -1,236 +1,133 @@ -"""Detector of the Displacement family (the families document): sudden jumps of corr_x / corr_y (px) of each target, at -the native cadence of the "advanced" export, and slow drift (end of file). A flipping target ("Target flipping") -belongs to the Target issue family: episodes() returns it, detect() leaves it to target_issue.py. +"""Displacement family: sudden jumps of the corrected displacements (corr_x, corr_y, px) of each target, and slow drift +(end of file). Global references are excluded, rejected measurements (empty corr) skipped. -1. Noise of the series = median of the gaps from one measurement to the next; jump = measurement K noises or more away - from the level before (median of the BASE previous measurements); the first BASE measurements, lacking a level - before, are compared with the level after (median of the BASE next ones), a gap there is a jump back at once. -2. Back (an hour of measurements in a row within half the jump: BASE at 15 min, a single one at 4 h; wf-improve - MICHIGAN, mod. 1) within a day, or one day later back at its usual place (first day of the export) or at the day - before's: "Sudden jump, back to normal", greyed; its cause is not established here (categories.py gives it the - cause of the category that overlaps it, otherwise: to be checked on the images). -3. Otherwise "Lasting shift", never greyed (maybe a real movement); less than a day of hindsight: "Jump, not back yet", - not greyed either. -4. Jumps following each other within GAP, all targets together, make a single episode (common mode); a single lasting - (or too recent) target is enough not to grey the episode. -5. Global references excluded; rejected measurements (empty corr) skipped. -6. Common peak (NORDEND, batch 01): at the same instant, at least PEAK_MIN targets move away by at least K_PEAK noises - from both neighbouring measurements, on the same side (the whole image shaken for one step): a jump back for each, - even under K. It joins the episodes of point 4 (next to a lasting jump, it greys nothing). -7. (CLARIDEN, batch 01) a jump that never comes back no longer ends the reading of the series, its new level becomes - the level before; an episode that is not greyed ends at its last jump, not at the returns; a flipping target - (REPEAT jumps back or more, each within a day of the previous one) makes a "back to normal" episode on its own, from - the first to the last flip, outside the common episodes. A target that flips for at least FLIP, with at most - MAX_FLIPPING targets at the same time, becomes "Target flipping" (Target issue, D-074); a lasting jump of at least - LARGE of the targets, "Large shift" (everything moved), always to be checked. wf-improve session Target issue, mod. 1, - S2: FLIP from 6 h to 36 h; a target flipping for a day during a nuisance of the whole scene (rain, fog, night of poor - visibility) is not a problem target; its jumps back go to the nuisance overlapping them, otherwise stay - Displacement. -8. Jump under K that stays (NIEDERHORN, batch 01): a jump of at least K_LASTING noises counts if it is lasting in the - sense of point 3 and the median of the day after moves away from that of the day before, on the jump's side, by at - least PLATEAU times the ordinary variation of the level from one UTC day to the next (median of the gaps between - daily levels); otherwise it is ignored (neither back nor too recent: only jumps of K noises or more are). +1. Jump: a measurement at least K noises (median step of the series) away from the level before (median of the BASE + previous measurements). +2. Back to normal: within BACK_H, an hour of measurements in a row (BASE at most, one at least) within half the jump of + the level before; or the day after the jump at the level of the day before (the jump ended a dip). Otherwise + "Lasting shift", or "Jump, not back yet" with less than BACK_H of hindsight. After a jump that never comes back, its + new level becomes the level before. +3. Jumps of all the targets less than GAP apart make one episode; a target that jumps and comes back at least REPEAT + times, each within BACK_H of the previous one, makes an episode of its own, from its first jump to its last return. +4. "Sudden jump, back to normal" is greyed and its cause is not established here: categories.py gives it the family of + an overlapping episode (rain, something passing...), otherwise it is to be checked on the images. A shift is never + greyed (maybe a real movement); its episode ends at its last jump. """ import statistics from datetime import timedelta -import data +from common import episode, noise, targets +from data import is_reference -K = 20 # tuned on E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 (17 and 19 - # noises) are caught by visibility.py: do not lower K -K_LASTING = 7 # under K, a jump that stays counts from K_LASTING noises (NIEDERHORN, batch 01: e09, corr_y −0.18 px at - # once, 8 noises, stayed 5 days) -PLATEAU = 8 # ... if it stays: median of the day after, on the jump's side, PLATEAU times the ordinary variation of the - # level from one day to the next away from the day before (e09: 13; E-001, healthy targets: 6 at most) -K_PEAK = 6 # one-step peak common to several targets (NORDEND shakes, 6 to 11 noises, batch 01) -PEAK_MIN = 3 # targets peaking at the same step for a common peak (one or two: ordinary noise) -BASE = 4 # measurements that make the level before (1 h at 15 min) -GAP = timedelta(hours=1) # two jumps within an hour, all targets together: a single episode -BACK_H = 24 # back within a day = artefact (E-002: plateaus up to 11.5 h) -FLIP = timedelta(hours=36) # a problem target flips for several days (PR_BG: 2.5 days); less is an agitated morning - # (E-009: C8 and C13, 5 flips in 1 h 20 during the movement) or a night of nuisance (NORDEND, JUNGFRAU: - # 12 to 28 h within a loss of visibility), a jump back, absorbed by its host -MAX_FLIPPING = 2 # targets flipping at the same time, at most (FLORIDA: the two stuck ones; E-002, cobweb on the - # references: 15 at once) -LARGE = 0.5 # share of the targets of a lasting jump for a large shift (E-004: almost all) -REPEAT = 5 # jumps back of the same target, each within a day of the other, that make it a flipping target (PR_BG of - # CLARIDEN, batch 01: about thirty in 2.5 days) +K = 20 # E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 are visibility's +BASE = 4 # measurements of the level before (1 h at 15 min); the return is judged on an hour (wf-improve MICHIGAN, mod. 1) +GAP = timedelta(hours=1) +BACK_H = timedelta(hours=24) # E-002: plateaus up to 11.5 h +REPEAT = 5 # CLARIDEN, batch 01: PR_BG flips 28 times in 2.5 days, the other targets 3 at most +JUMP_BACK = "Sudden jump, back to normal" def _jumps(t, v): - """A series (instants, values) -> [(start i, end i, state)], state 'back', 'lasting' or 'recent'.""" - day = timedelta(hours=BACK_H) - noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:])) - step = statistics.median(b - a for a, b in zip(t, t[1:])) # cadence of the series - back_n = max(1, min(BASE, round(timedelta(hours=1) / step))) # measurements of an hour (BASE at 15 min, one at 4 h) - # ponytail: usual place = first day of the export, wrong if it starts within an artefact - ref = statistics.median(x for x, u in zip(v, t) if u < t[0] + day) - lv = {} # level of each UTC day; its ordinary variation from one day to the next judges a jump under K - for x, u in zip(v, t): - lv.setdefault(u.date(), []).append(x) - lv = [statistics.median(x) for _, x in sorted(lv.items())] - ordinary = statistics.median(abs(b - a) for a, b in zip(lv, lv[1:])) if len(lv) > 2 else 0 - # the first BASE measurements, lacking a level before, face the level after: a gap there came back at once - out = [(k, k, "back") for k in range(min(BASE, len(v) - BASE)) - if noise and abs(v[k] - statistics.median(v[k + 1:k + 1 + BASE])) >= K * noise] - i = BASE - while noise and i < len(v): + """A series (instants, values) -> [(start index, end index, state)], state "back", "lasting" or "recent".""" + n = noise(v) + back_n = max(1, min(BASE, round(timedelta(hours=1) / statistics.median(b - a for a, b in zip(t, t[1:]))))) + out, i = [], BASE + while n and i < len(v): base = statistics.median(v[i - BASE:i]) - if abs(v[i] - base) < K_LASTING * noise: + if abs(v[i] - base) < K * n: i += 1 continue - tol = abs(v[i] - base) / 2 # back: an hour of measurements in a row within half the jump from the level before + tol = abs(v[i] - base) / 2 j = next((k for k in range(i + 1, len(v)) if all(abs(x - base) < tol for x in v[k:k + back_n])), len(v)) - before = statistics.median([x for x, u in zip(v, t) if t[i] - day <= u < t[i]] or [base]) - after = statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + day) - if j < len(v) and t[j] - t[i] <= day: + if j < len(v) and t[j] - t[i] <= BACK_H: state = "back" - elif abs(after - ref) < tol or abs(after - before) < tol: # the day after, the target is at its usual place, - state, j = "back", i + 1 # or where it was the day before: the jump ended a dip + elif abs(statistics.median(x for x, u in zip(v, t) if t[i] <= u < t[i] + BACK_H) + - statistics.median([x for x, u in zip(v, t) if t[i] - BACK_H <= u < t[i]] or [base])) < tol: + state, j = "back", i + 1 # the day after at the level of the day before: the jump ended a dip else: - state = "lasting" if t[-1] - t[i] >= day else "recent" # less than a day of hindsight: not known yet - if abs(v[i] - base) < K * noise and (state != "lasting" or (after - before) * (v[i] - base) < 0 - or not ordinary or abs(after - before) < PLATEAU * ordinary): - i += 1 # under K, only a jump that stays counts: the day after far from the day before, on its side - continue + state = "lasting" if t[-1] - t[i] >= BACK_H else "recent" out.append((i, j - 1, state)) - # level before rebuilt after the return; never back: the new level becomes the level before, the rest is - # still read (PR_BG of CLARIDEN, batch 01: its flips after the 10:30 jump were invisible) i = (j if j < len(v) else i) + BASE return out -def _peaks(t, v): - """Instants where the measurement moves away by at least K_PEAK noises from both neighbours, same side (one-step peak).""" - noise = statistics.median(abs(y - x) for x, y in zip(v, v[1:])) - return [t[i] for i in range(1, len(v) - 1) if noise and (v[i] - v[i - 1]) * (v[i] - v[i + 1]) > 0 - and min(abs(v[i] - v[i - 1]), abs(v[i] - v[i + 1])) >= K_PEAK * noise] - - -def episodes(measurements): - """{sensor: [measurement]} (data.load) -> jump episodes, sorted by start: Displacement, and Target issue for a - flipping target.""" - found = [] # (start, end, state, sensor) - peaks = {} # instant -> targets peaking there for one step - for name, ms in measurements.items(): - if data.is_reference(ms): - continue - for axis in ("corr_x", "corr_y"): - kept = [m for m in ms if m[axis] is not None] - if len(kept) <= BASE: - continue - t, v = [m["t"] for m in kept], [m[axis] for m in kept] - # a jump that did not come back ends with itself, not with the data (its bar hid the others) - found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)] - for u in _peaks(t, v): - peaks.setdefault(u, set()).add(name) - found += [(u, u, "back", s) for u, cs in peaks.items() if len(cs) >= PEAK_MIN for s in cs] # common peak: back - alone = _flips(found) # a target that keeps jumping and coming back: an episode on its own, outside the common ones - found = [f for f in found if f[2] != "back" or not any(f[3] in g["sensors"] and g["start"] <= f[0] <= g["end"] - for g in alone)] - groups = [] # a jump that did not come back only counts by its instant: its end is often the end of the data - for s, e, state, name in sorted(found): - if groups and s <= groups[-1]["until"] + GAP: - g = groups[-1] - else: - g = {"start": s, "end": e, "until": s, "last": s, "sensors": {}} - groups.append(g) - g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s - g["sensors"].setdefault(name, set()).add(state) - long = [g for g in alone if g["flip"]] # flips of many targets at the same time: the compensation, not a - for g in long: # problem target - if sum(h["start"] <= g["end"] and h["end"] >= g["start"] for h in long) > MAX_FLIPPING: - g["flip"] = 0 - total = sum(not data.is_reference(ms) for ms in measurements.values()) - return sorted((_episode(g, total) for g in groups + alone), key=lambda ep: ep["start"]) - - -def _flips(found): - """Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> a group, from the first - to the last flip.""" - day, alone = timedelta(hours=BACK_H), [] +def _repeats(found): + """Jumps back of the same target, each within BACK_H of the previous one: REPEAT or more -> [(start, end, target)].""" + out = [] for name in {f[3] for f in found}: - chain = [] # (start, end); corr_x and corr_y twice: counted by instant + chain = [] # (start, end); corr_x and corr_y together: counted by instant for s, e in sorted({(s, e) for s, e, state, n in found if n == name and state == "back"}) + [(None, None)]: - if chain and s is not None and s <= max(c[1] for c in chain) + day: + if chain and s is not None and s <= max(c[1] for c in chain) + BACK_H: chain.append((s, e)) continue if len({c[0] for c in chain}) >= REPEAT: - alone.append({"start": chain[0][0], "end": max(c[1] for c in chain), "sensors": {name: {"back"}}, - "flip": len({c[0] for c in chain}) if max(c[1] for c in chain) - chain[0][0] >= FLIP else 0}) + out.append((chain[0][0], max(c[1] for c in chain), name)) chain = [(s, e)] - return alone + return out -def _episode(g, total): - """A group -> episode; a single lasting (or too recent) jump is enough not to grey; a flipping target: "Target - flipping" (Target issue, D-074); a lasting jump of LARGE targets or more: "Large shift".""" - n = len(g["sensors"]) - lasting = sum("lasting" in st for st in g["sensors"].values()) - recent = any("recent" in st for st in g["sensors"].values()) - who = "One target" if n == 1 else f"{n} targets together (a single event)" - if g.get("flip"): - name = next(iter(g["sensors"])) - return {"start": data.iso(g["start"]), "end": data.iso(g["end"]), "kind": "Target issue", "label": "Target flipping", - "text": f"{name} jumped and came back {g['flip']} times: its tracking is not reliable, look at the images " - "of the target.", "ignore": True, "sensors": [name], "lasting": []} - if lasting and n >= LARGE * total: - label = "Large shift" - text = f"{n} targets jumped together and {lasting} of them stayed shifted: everything moved, please check it." - elif lasting: - label = "Lasting shift" - some = "" if lasting == n else f", {lasting} of them stayed shifted for more than a day" - text = (f"{who} jumped{some}" if some else f"{who} jumped and stayed shifted for more than a day") + \ - ": this may be a real movement, please check it." - elif recent: - label = "Jump, not back yet" - text = f"{who} jumped less than a day before the last measurement and has not come back yet: check again later." +def _episode(start, end, states): + """{target: {states}} -> the episode of a group of jumps.""" + n = len(states) + lasting = sum("lasting" in st for st in states.values()) + who = targets(n).capitalize() + if lasting: + some = "stayed shifted" if lasting == n else f"{lasting} of them stayed shifted" + label, text = "Lasting shift", f"{who} shifted; {some}." + elif any("recent" in st for st in states.values()): + label, text = "Jump, not back yet", f"{who} shifted and {'has' if n == 1 else 'have'} not come back yet." else: - label = "Sudden jump, back to normal" - text = f"{who} jumped and came back within a day: the cause is not established, look at the images." - end = g["last"] if lasting or recent else g["end"] # not greyed: the last jump, not the returns - return {"start": data.iso(g["start"]), "end": data.iso(end), "kind": "Displacement", "label": label, - "text": text, "ignore": not lasting and not recent, "sensors": sorted(g["sensors"]), - "lasting": sorted(s for s, st in g["sensors"].items() if "lasting" in st)} # stayed shifted (event sentence) + return episode(start, end, "Displacement", JUMP_BACK, f"{who} jumped and came back. The cause is not established: " + "look at the images.", states, True) + return episode(start, end, "Displacement", label, text + " Check whether the structure moved: look at the images.", + states, False) + + +def jumps(measurements): + found = [] # (start, end, state, target) + for name, ms in measurements.items(): + if is_reference(ms): + continue + for axis in ("corr_x", "corr_y"): + kept = [m for m in ms if m[axis] is not None] + if len(kept) > BASE: + t, v = [m["t"] for m in kept], [m[axis] for m in kept] + found += [(t[a], t[b] if state == "back" else t[a], state, name) for a, b, state in _jumps(t, v)] + alone = _repeats(found) + out = [_episode(a, b, {name: {"back"}}) for a, b, name in alone] + groups = [] # a shift only counts by its instant: its end is often the end of the data + for s, e, state, name in sorted(found): + if state == "back" and any(name == n and a <= s <= b for a, b, n in alone): + continue + if not groups or s > groups[-1]["until"] + GAP: + groups.append({"start": s, "end": e, "until": s, "states": {}}) + g = groups[-1] + g["end"], g["until"], g["last"] = max(g["end"], e), max(g["until"], e if state == "back" else s), s + g["states"].setdefault(name, set()).add(state) + for g in groups: + shift = any(st != {"back"} for st in g["states"].values()) + out.append(_episode(g["start"], g["last"] if shift else g["end"], g["states"])) + return out def detect(measurements): - return sorted(episodes(measurements) + drift(measurements), - key=lambda ep: ep["start"]) + return jumps(measurements) + drift(measurements) -# --- Slow drift ------------------------------------------------------------------------------------------------------ -# A target that moves little by little, day after day. Takes up the trend of the Displacements tab (D-055) on the -# "advanced" export (corr_x, corr_y, px). Method, per target (global references excluded) and per axis: -# 1. Level of the day: median of the measurements of each UTC day with at least HOURS measured hours out of 24, as a -# share of the hours the station measures (wf-improve MICHIGAN, mod. 1: at one measurement every 4 h, 5 of the 6 hours; -# before, no day counted); at least DAYS days. -# 2. Theil-Sen slope and Kendall tau of the daily levels; amplitude of the day: gap between its 9th and 1st deciles. -# 3. Drift: |slope| × duration ≥ SIZE times the median daily amplitude, |tau| ≥ STEADY, and no step from one day to the -# next of more than half the drift (a jump that lasts is not a drift: the jump rule takes it). -# 4. One episode per target (one axis is enough), from the first measurement of the first day to the last of the last -# day; not greyed. -# 4b. Piecewise (wf-improve session Displacement, mod. 1, S1): a target without a drift over the whole window may drift for -# a few days, then settle or go back. On each run of at least DAYS kept days, drift = level of the last day minus -# that of the first, at least PIECE daily amplitudes (median of the series), |tau| ≥ STEADY with the same sign, no -# step from one kept day to the next of more than half the drift; pieces taken from the largest drift to the -# smallest, without overlapping (they may share the day where the level turns): each goes from one extreme of the -# level to the other, one episode per piece and per axis. At PIECE = 1, the thermal cycles of a few days and the -# noise of the levels get through (JUNGFRAU, NORDEND): 3.5 keeps the rows of CLARIDEN (3.8 to 8.3 amplitudes), -# drops JUNGFRAU (3.1 at most). The pieces do not replace step 3. A piece does not cross a hole: two consecutive kept -# days are at most MAX_HOLE days apart (mod. 2: the gap of a level across three days without a measurement is a -# step, not a slope; it made pieces of the whole window, CLARIDEN 09-28 → 11-01). A missing day does not cut. -# 5. Camera drift (same mod.): at least LARGE of the targets drift on the same side on corr_x or corr_y, and at least -# LARGE of the sensors, references included, on their raw positions x or y: only the pieces of point 4b are -# reported, not the trend over the whole window (mod. 1, S2: the badly compensated drifting camera, NEBRASKA, is a -# Displacement for the labels). The targets of a bridge following the season (JUNGFRAU, labelled Thermal -# displacement day after day) pass step 3 but not PIECE; NEBRASKA both. -HOURS = 20 # a day counts if it has at least 20 measured hours out of 24 (as D-055), as a share of the measured hours -DAYS = 5 # minimum days for a trend (D-055) -SIZE = 1.0 # drift ≥ SIZE daily amplitudes (D-055) -STEADY = 0.6 # |Kendall tau| of the daily levels (D-055) -PIECE = 3.5 # piecewise drift ≥ PIECE daily amplitudes (point 4b) -MAX_HOLE = 2 # a piece does not go from one kept day to the next more than MAX_HOLE days apart (point 4b) +# --- Slow drift: a target that moves little by little, day after day (the trend of the Displacements tab, D-055). +# Per target and per axis (corr_x, corr_y): the level of each UTC day measured at least HOURS of the hours the station +# measures (median of the day), at least DAYS days; the amplitude of a day: its 9th minus its 1st decile. +# 1. Over the whole window: Theil-Sen slope × duration at least SIZE median daily amplitudes, |Kendall tau| ≥ STEADY, and +# no step from one day to the next larger than half the drift (that is a jump). +# 2. Otherwise, pieces: runs of at least DAYS kept days whose last level minus the first is at least PIECE amplitudes, with +# the same tau and step conditions; taken from the largest to the smallest without overlapping (wf-improve +# Displacement, mod. 1, S1: rows of CLARIDEN, 3.8 to 8.3 amplitudes; thermal cycles of JUNGFRAU, 3.1 at most). +# One episode per drift, not greyed: the largest of the two axes over the whole window, otherwise each piece. +HOURS = 20 # out of 24, as a share of the hours the station measures (wf-improve MICHIGAN, mod. 1) +DAYS = 5 # D-055 +SIZE = 1.0 # D-055 +STEADY = 0.6 # D-055 +PIECE = 3.5 # wf-improve Displacement, mod. 1, S1 def _decile(vs, q): @@ -239,50 +136,34 @@ def _decile(vs, q): def _days(ms, col): - """-> {UTC day: its measurements}, the kept days (point 1).""" + """-> (kept days, their levels, median daily amplitude, [(first instant, last) of each day]).""" days = {} for m in ms: if m[col] is not None: days.setdefault(m["t"].date(), []).append(m) slots = len({m["t"].hour for dm in days.values() for m in dm}) # hours the station measures (6 at a 4 h step) - return {d: dm for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots} + ds = sorted(d for d, dm in days.items() if len({m["t"].hour for m in dm}) >= HOURS / 24 * slots) + vals = [[m[col] for m in days[d]] for d in ds] + ampl = statistics.median(_decile(v, 0.9) - _decile(v, 0.1) for v in vals) if ds else 0 + return ds, [statistics.median(v) for v in vals], ampl, [(days[d][0]["t"], days[d][-1]["t"]) for d in ds] -def _amplitude(days, col): - return statistics.median(_decile([m[col] for m in dm], 0.9) - _decile([m[col] for m in dm], 0.1) for dm in days.values()) +def _whole(ds, lv, ampl, _): + """-> [(drift, 0, last index)] if the levels drift over the whole window, else [].""" + pairs = [(i, j) for i in range(len(ds)) for j in range(i + 1, len(ds))] + slope = statistics.median((lv[j] - lv[i]) / (ds[j] - ds[i]).days for i, j in pairs) + tau = sum((lv[j] > lv[i]) - (lv[j] < lv[i]) for i, j in pairs) / len(pairs) + d = slope * (ds[-1] - ds[0]).days + step = max(abs(b - a) for a, b in zip(lv, lv[1:])) + return [(d, 0, len(ds) - 1)] if abs(d) >= SIZE * ampl and abs(tau) >= STEADY and step <= abs(d) / 2 else [] -def _drift(ms, col): - """-> (drift in px, days, first instant, last) or None.""" - days = _days(ms, col) - if len(days) < DAYS: - return None - lv = sorted((d.toordinal(), statistics.median(m[col] for m in dm)) for d, dm in days.items()) - pairs = [(a, b) for k, a in enumerate(lv) for b in lv[k + 1:]] - slope = statistics.median((b[1] - a[1]) / (b[0] - a[0]) for a, b in pairs) - tau = sum((b[1] > a[1]) - (b[1] < a[1]) for a, b in pairs) / len(pairs) - drift = slope * (lv[-1][0] - lv[0][0]) - step = max(abs(b[1] - a[1]) for a, b in zip(lv, lv[1:])) - if abs(drift) < SIZE * _amplitude(days, col) or abs(tau) < STEADY or step > abs(drift) / 2: - return None - d0, d1 = min(days), max(days) - return drift, len(days), min(m["t"] for m in days[d0]), max(m["t"] for m in days[d1]) - - -def _pieces(ms, col): - """-> [(drift in px, days, first instant, last)]: the drift pieces (point 4b).""" - days = _days(ms, col) - if len(days) < DAYS: - return [] - ds = sorted(days) - lv = [statistics.median(m[col] for m in days[d]) for d in ds] - ampl = _amplitude(days, col) - pieces = [] # (drift, i, j) +def _pieces(ds, lv, ampl, _): + """-> [(drift, first index, last index)]: the drift pieces, without overlapping.""" + pieces = [] for i in range(len(ds)): signs = step = 0 for j in range(i + 1, len(ds)): - if (ds[j] - ds[j - 1]).days > MAX_HOLE: - break # a gap across a hole is a step (the jump rule takes it), not a slope signs += sum((lv[j] > lv[k]) - (lv[j] < lv[k]) for k in range(i, j)) # Kendall tau, step by step step = max(step, abs(lv[j] - lv[j - 1])) n, d = j - i + 1, lv[j] - lv[i] @@ -293,26 +174,17 @@ def _pieces(ms, col): for d, i, j in sorted(pieces, key=lambda x: (-abs(x[0]), x[2] - x[1])): if all(j <= a or i >= b for _, a, b in taken): taken.append((d, i, j)) - return [(d, j - i + 1, min(m["t"] for m in days[ds[i]]), max(m["t"] for m in days[ds[j]])) for d, i, j in taken] - - -def _common(measurements, cols): - """At least LARGE of the sensors drift on the same side on the same axis of cols.""" - for col in cols: - signs = [x[0] > 0 for ms in measurements.values() if (x := _drift(ms, col))] - if max(signs.count(True), signs.count(False)) >= LARGE * len(measurements): - return True - return False + return taken def drift(measurements): - targets = {s: ms for s, ms in measurements.items() if not data.is_reference(ms)} - camera = _common(targets, ("corr_x", "corr_y")) and _common(measurements, ("x", "y")) # point 5: pieces only out = [] - for s, ms in sorted(targets.items()): - r = [] if camera else [x for col in ("corr_x", "corr_y") if (x := _drift(ms, col))] - r = [max(r, key=lambda x: abs(x[0]))] if r else [x for col in ("corr_x", "corr_y") for x in _pieces(ms, col)] - out += [{"start": data.iso(a), "end": data.iso(b), "kind": "Displacement", "label": "Slow drift", - "text": f"{s} drifts slowly and steadily: {d:+.2f} px in {n} days. Keep watching it.", - "ignore": False, "sensors": [s]} for d, n, a, b in r] + for s, ms in sorted(measurements.items()): + if is_reference(ms): + continue + axes = [a for col in ("corr_x", "corr_y") if len((a := _days(ms, col))[0]) >= DAYS] + whole = [(w, a) for a in axes for w in _whole(*a)] + found = [max(whole, key=lambda x: abs(x[0][0]))] if whole else [(p, a) for a in axes for p in _pieces(*a)] + out += [episode(a[3][i][0], a[3][j][1], "Displacement", "Slow drift", f"{s} drifts slowly and steadily: {d:+.2f} " + f"px in {j - i + 1} days. Keep watching it.", [s], False) for (d, i, j), a in found] return out