Essai du 2026-11-05 : le batch, le scan, la première séance d'amélioration, l'audit du scanner et ses outils
batches/2026-11-05/ : détecteurs, vérité, rapport du scanner, improve/ (mesures, modification, validation, bilan, tools/instants.py de l'upgrader) ; docs/essais/ : le journal de l'essai et l'audit ; outils/ : les scripts de lecture de l'audit, réutilisables (JOUR=AAAA-MM-JJ). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
dab71a9037
commit
ed4cedbe70
47 files changed
+1490
No files matched your search
@@ -0,0 +1,37 @@
|
||||
# Agitation d'une station sur un jour (STATION JOUR [HH HH]) : par instant, médiane sur les cibles du |Δcorr| d'un pas (en bruits), de l'error_value face à la nuit, de la sharpness et du flux face à la nuit ; pose de la caméra.
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from outils import RACINE, station, bruit
|
||||
from data import load_camera
|
||||
|
||||
nom, jour = sys.argv[1:3]
|
||||
h0, h1 = (int(x) for x in sys.argv[3:5]) if len(sys.argv) > 4 else (0, 24)
|
||||
d = datetime.fromisoformat(jour).replace(tzinfo=timezone.utc)
|
||||
m, refs = station(nom)
|
||||
nuit = lambda x: d <= x["t"] < d + timedelta(hours=5) # noqa: E731 ponytail: nuit = 00-05 UTC, juste pour une base
|
||||
cibles = [s for s in m if s not in refs]
|
||||
base = {}
|
||||
for s in cibles:
|
||||
for c in ("error_value", "sharpness", "flux"):
|
||||
v = [x[c] for x in m[s] if nuit(x) and x[c] is not None]
|
||||
if v and statistics.median(v):
|
||||
base[s, c] = statistics.median(v)
|
||||
bx = {s: bruit(m[s], "corr_x") for s in cibles}
|
||||
par = {}
|
||||
for s in cibles:
|
||||
ms = [x for x in m[s] if d + timedelta(hours=h0) <= x["t"] < d + timedelta(hours=h1)]
|
||||
for a, b in zip(ms, ms[1:]):
|
||||
r = par.setdefault(b["t"], {"d": [], "e": [], "sh": [], "fl": []})
|
||||
if a["corr_x"] is not None and b["corr_x"] is not None and bx[s]:
|
||||
r["d"].append(abs(b["corr_x"] - a["corr_x"]) / bx[s])
|
||||
for c, k in (("error_value", "e"), ("sharpness", "sh"), ("flux", "fl")):
|
||||
if b[c] is not None and (s, c) in base:
|
||||
r[k].append(b[c] / base[s, c])
|
||||
pose = {p["t"]: p for p in load_camera(str(RACINE / "data" / nom))}
|
||||
med = lambda v: f"{statistics.median(v):5.2f}" if v else " - " # noqa: E731
|
||||
print("t |Δx|/bruit ev/nuit sharp/nuit flux/nuit pan tilt")
|
||||
for t in sorted(par):
|
||||
r, p = par[t], pose.get(t, {})
|
||||
print(t.strftime("%H:%M"), med(r["d"]), med(r["e"]), med(r["sh"]), med(r["fl"]),
|
||||
f"{p.get('pan') or float('nan'):+.5f} {p.get('tilt') or float('nan'):+.5f}")
|
||||
@@ -0,0 +1,16 @@
|
||||
# Séries brutes de quelques capteurs : STATION CAPTEUR[,CAPTEUR…] DE A (AAAA-MM-JJTHH:MM) -> t, corr_x, corr_y, error_value/seuil, rejet ; bruit en tête.
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from outils import station, bruit, rejected
|
||||
|
||||
nom, noms, de, a = sys.argv[1], sys.argv[2].split(","), *sys.argv[3:5]
|
||||
de, a = (datetime.fromisoformat(x).replace(tzinfo=timezone.utc) for x in (de, a))
|
||||
m, refs = station(nom)
|
||||
for s in noms:
|
||||
ms = m[s]
|
||||
print(f"== {s} {'(réf)' if s in refs else ''} bruit x={bruit(ms, 'corr_x'):.4g} y={bruit(ms, 'corr_y'):.4g}")
|
||||
for x in ms:
|
||||
if de <= x["t"] <= a:
|
||||
ev = x["error_value"] / x["error_threshold"] if x["error_threshold"] and x["error_value"] is not None else float("nan")
|
||||
f = lambda v: " None " if v is None else f"{v:+8.3f}" # noqa: E731
|
||||
print(x["t"].strftime("%m-%d %H:%M"), f(x["corr_x"]), f(x["corr_y"]), f"{ev:.2f}", "REJ" if rejected(x) else "")
|
||||
@@ -0,0 +1,10 @@
|
||||
# Les épisodes bruts de visibility.episodes (avant categories.py) d'une station sur le jour, avec 3 jours d'historique : STATION.
|
||||
import sys
|
||||
from outils import D, B, H, station, iso
|
||||
import visibility
|
||||
|
||||
m, _ = station(sys.argv[1])
|
||||
morceau = {s: k for s, ms in m.items() if (k := [x for x in ms if H <= x["t"] < B])}
|
||||
for e in visibility.episodes(morceau):
|
||||
if e["end"] >= iso(D):
|
||||
print(e["start"][11:], e["end"][11:], e["kind"], e["label"], len(e["sensors"]), e.get("level", ""))
|
||||
@@ -0,0 +1,38 @@
|
||||
# Instant par instant (STATION [HH:MM HH:MM] [--pas MIN]) : capteurs mesurés, rejetés (cibles, références), les cibles les plus écartées en bruits.
|
||||
import argparse
|
||||
import statistics
|
||||
from outils import D, B, station, ecarts, hhmm, rejected
|
||||
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("station")
|
||||
p.add_argument("de", nargs="?", default="00:00")
|
||||
p.add_argument("a", nargs="?", default="23:59")
|
||||
p.add_argument("--pas", type=float, help="bruit sur les paires espacées d'au moins PAS minutes (rafales)")
|
||||
p.add_argument("--k", type=int, default=4)
|
||||
p.add_argument("--seuil", type=float, default=5, help="n'afficher que les cibles au-delà (bruits)")
|
||||
a = p.parse_args()
|
||||
|
||||
m, refs = station(a.station)
|
||||
ex = {s: ecarts(ms, "corr_x", a.pas) for s, ms in m.items() if s not in refs}
|
||||
ey = {s: ecarts(ms, "corr_y", a.pas) for s, ms in m.items() if s not in refs}
|
||||
par = {}
|
||||
for s, ms in m.items():
|
||||
for x in ms:
|
||||
if D <= x["t"] < B:
|
||||
par.setdefault(x["t"], {})[s] = x
|
||||
for t in sorted(par):
|
||||
if not (a.de <= hhmm(t) <= a.a):
|
||||
continue
|
||||
c = par[t]
|
||||
rej = sorted(s for s, x in c.items() if rejected(x))
|
||||
rr = [s for s in rej if s in refs]
|
||||
rc = [s for s in rej if s not in refs]
|
||||
ev = [x["error_value"] / x["error_threshold"] for x in c.values() if x["error_value"] is not None and x["error_threshold"]]
|
||||
top = []
|
||||
for nom, e in (("x", ex), ("y", ey)):
|
||||
v = sorted(((abs(e[s][t]), s, e[s][t]) for s in e if e[s].get(t) is not None), reverse=True)
|
||||
top += [f"{s}:{nom}{z:+.0f}" for _, s, z in v[:a.k] if abs(z) > a.seuil]
|
||||
n = sum(1 for z, _, _ in v if z > a.seuil)
|
||||
top.append(f"[{nom}>{a.seuil:g}: {n}/{len(v)}]")
|
||||
print(f"{hhmm(t)} n={len(c):2d} ev/s={statistics.median(ev) if ev else float('nan'):.2f} "
|
||||
f"rejC={len(rc)} rejR={rr} | {' '.join(top)}")
|
||||
@@ -0,0 +1,44 @@
|
||||
# Les instants anormaux qu'aucune étiquette de la vérité (hors Thermal) ne couvre, toutes stations : rejets non habituels, ou cibles qui sautent face à leur niveau local (±6 mesures), en bruits.
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from outils import D, B, RACINE, station, bruit, hhmm, rejected, iso
|
||||
|
||||
BATCH = RACINE / "batches" / iso(D)[:10]
|
||||
K, N = 10, 3 # un saut : au moins N cibles à plus de K bruits de leur niveau local
|
||||
stations = sys.argv[1:] or sorted(p.stem for p in (BATCH / "verite").glob("*.jsonl"))
|
||||
for nom in stations:
|
||||
labels = [json.loads(l) for l in open(BATCH / "verite" / f"{nom}.jsonl") if l.strip()]
|
||||
labels = [e for e in labels if e["categorie"] not in ("Thermal displacement",)]
|
||||
m, refs = station(nom)
|
||||
jour = {s: [x for x in ms if D <= x["t"] < B] for s, ms in m.items()}
|
||||
habituel = {s for s, ms in jour.items() if ms and sum(map(rejected, ms)) > 0.5 * len(ms)}
|
||||
par = {}
|
||||
for s, ms in jour.items():
|
||||
if s in refs:
|
||||
continue
|
||||
for col in ("corr_x", "corr_y"):
|
||||
b = bruit(m[s], col, 10)
|
||||
if not b:
|
||||
continue
|
||||
for i, x in enumerate(ms):
|
||||
voisins = [y[col] for y in ms[max(0, i - 6):i] + ms[i + 1:i + 7] if y[col] is not None]
|
||||
if x[col] is not None and len(voisins) >= 4 and abs(x[col] - statistics.median(voisins)) > K * b:
|
||||
par.setdefault(x["t"], set()).add(s)
|
||||
rej = {}
|
||||
for s, ms in jour.items():
|
||||
for x in ms:
|
||||
if rejected(x) and s not in habituel:
|
||||
rej.setdefault(x["t"], set()).add(s)
|
||||
sorties = []
|
||||
for t in sorted(set(par) | set(rej)):
|
||||
sauts, r = par.get(t, set()), rej.get(t, set())
|
||||
if len(sauts) < N and not r:
|
||||
continue
|
||||
if any(e["debut"] <= iso(t) <= e["fin"] for e in labels):
|
||||
continue
|
||||
sorties.append(f"{hhmm(t)} sauts={len(sauts)} rejets={sorted(r)[:4]}{'…' if len(r) > 4 else ''}")
|
||||
print(f"== {nom} (rejet habituel : {sorted(habituel)}) : {len(sorties)} instants non couverts")
|
||||
for l in sorties[:80]:
|
||||
print(" ", l)
|
||||
@@ -0,0 +1,52 @@
|
||||
# Outils communs de l'audit : charge une station (chargeur des détecteurs), écarts en bruits, événements des détecteurs.
|
||||
# Le jour : la variable JOUR (AAAA-MM-JJ, UTC), par défaut 2026-11-05 (l'essai d'où viennent ces scripts).
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
RACINE = Path(__file__).resolve().parents[1] # le dépôt Optimizer : data/, detectors/, batches/
|
||||
sys.path.insert(0, str(RACINE / "detectors" / "scripts"))
|
||||
from data import load, load_temperature, is_reference, rejected, iso # noqa: E402,F401
|
||||
|
||||
D = datetime.strptime(os.environ.get("JOUR", "2026-11-05"), "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
B = D + timedelta(days=1)
|
||||
H = D - timedelta(days=3)
|
||||
|
||||
|
||||
def station(nom):
|
||||
"""-> (mesures, refs) : {capteur: [mesure]} et l'ensemble des références."""
|
||||
m = load(str(RACINE / "data" / nom))
|
||||
return m, {s for s, ms in m.items() if is_reference(ms)}
|
||||
|
||||
|
||||
def bruit(ms, col, pas_min=None):
|
||||
"""Bruit propre : médiane des écarts entre deux mesures sur l'historique et le jour (comme batch.py) ;
|
||||
pas_min (minutes) : ne garder que les paires espacées d'au moins pas_min (contre les rafales)."""
|
||||
h = [(m["t"], m[col]) for m in ms if H <= m["t"] < B and m[col] is not None]
|
||||
e = [abs(b - a) for (ta, a), (tb, b) in zip(h, h[1:])
|
||||
if pas_min is None or (tb - ta).total_seconds() >= pas_min * 60]
|
||||
return statistics.median(e) if len(e) > 1 else 0
|
||||
|
||||
|
||||
def ecarts(ms, col, pas_min=None):
|
||||
"""{t: écart à la médiane du jour en bruits} pour le jour ; None si rejetée."""
|
||||
v = [m[col] for m in ms if D <= m["t"] < B and m[col] is not None]
|
||||
b = bruit(ms, col, pas_min)
|
||||
if not v or b <= 0:
|
||||
return {}
|
||||
med = statistics.median(v)
|
||||
return {m["t"]: None if m[col] is None else (m[col] - med) / b for m in ms if D <= m["t"] < B}
|
||||
|
||||
|
||||
def hhmm(t):
|
||||
return t.strftime("%H:%M")
|
||||
|
||||
|
||||
def evenements(nom):
|
||||
"""Les événements des détecteurs comme batch.py les obtient (3 jours d'historique)."""
|
||||
import categories
|
||||
m = load(str(RACINE / "data" / nom))
|
||||
morceau = {s: k for s, ms in m.items() if (k := [x for x in ms if H <= x["t"] < B])}
|
||||
return [v for v in categories.events(morceau, B, str(RACINE / "data" / nom)) if v["end"] > iso(D)]
|
||||
@@ -0,0 +1,13 @@
|
||||
# Rejets d'un capteur jour par jour sur tout l'historique : STATION CAPTEUR -> jour, mesures, rejetées.
|
||||
import sys
|
||||
from collections import Counter
|
||||
from outils import station, rejected
|
||||
|
||||
m, refs = station(sys.argv[1])
|
||||
n, r = Counter(), Counter()
|
||||
for x in m[sys.argv[2]]:
|
||||
d = x["t"].date().isoformat()
|
||||
n[d] += 1
|
||||
r[d] += rejected(x)
|
||||
print(sys.argv[2], "(réf)" if sys.argv[2] in refs else "(cible)")
|
||||
print(" ".join(f"{d[5:]}:{r[d]}/{n[d]}" for d in sorted(n)))
|
||||
Reference in new issue
Block a user