Files

408 lines
21 KiB
Python
Raw Permalink Normal View History

"""The six image-quality metrics (peak, flux, sharpness, fwhm, dominance, snr) per sensor, against its calm reference.
Usage: python3 tools/scripts/quality.py <folder> [--start T] [--end T] [--sensors A,B] [--calm T1/T2,T3/T4]
[--common] [--table METRIC] [--step 15min] [--threshold 0.5]
python3 tools/scripts/quality.py <folder> --labels labeling/auto/X.jsonl [--calm ...] [--start T] [--end T]
[--numbers 3,7] [--threshold 0.5] [--margin 60] [--edges] | [--outside [--min 1] [--except "Thermal displacement"]]
.venv/bin/python tools/scripts/quality.py <folder> --figure [--calm ...] [--start T] [--end T]
[--metrics flux,snr,dominance] [--labels F] [--zone T1/T2 ...] [--out D]
python3 tools/scripts/quality.py --self-test (check of the ratio computation)
Definitions: tools/DATA.md. Columns filled on recent cameras only; on a folder where
they are empty, the tool says so and stops.
Reference of a sensor: the median of each metric over its measurements in the --calm windows, at the same UTC hour of the day
(24 slots; a slot without a calm measurement takes the median of all calm hours). Without --calm: the whole folder.
Each measurement becomes a ratio value / reference: 1 = like the reference; < 1 lower; > 1 higher. For fwhm and
dominance, it is the rise that says a degraded image (wider spot, stronger second peak). Rejected measurements included
(the quality of their image is still measured).
Without --labels or --figure:
1. the maximum peak of the folder, then the summary per sensor: rejected measurements, "pk/max" (median peak of the sensor over the
window divided by this maximum: a spot at the ceiling of the image has a constant peak), and for each metric the median ratio over the window and the extreme (min for peak, flux, sharpness, snr; max for
fwhm, dominance) with its time; then the lines "targets" and "references": the median of the median ratios;
2. --common: per step of --step, the median of the ratios of the targets and that of the references, for each metric;
3. --table M: per step, the ratio of M of each sensor ("*": a measurement of the step rejected), "med" (median over the
sensors) and the number of sensors below --threshold (above 1/--threshold for fwhm and dominance).
--labels F (one JSON per line, start, end, family), without --figure: per label (numbered in the order of the
file; --start/--end keep those that start within), the median ratio of the targets and of the references over the hour
before, the range and the hour after (--margin minutes): "tgt 0.95|0.40|0.97"; then the sensors "affected" over the
range (flux or snr below --threshold, or dominance above 1/--threshold), with their worst flux ratio, from lowest to
highest (r = reference). --edges: per time of the range widened by --margin, the number of affected sensors and the median
of the targets of flux, snr, dominance ("<": within the range). --outside: instead, the runs of times from --start to --end
that no label covers (except families --except), where at least --min sensors are affected (times less than 15 min apart
grouped), with the affected sensors and their worst flux.
--figure (matplotlib): PNG "quality-common": one panel per metric, logarithmic scale, median of the targets (black),
of the references (red) and band from the 10th to the 90th percentile of the targets; PNG "quality-<metric>" per --metrics: one small
plot per sensor, references in red. --labels shades each label in one colour per family; --zone shades
ranges. Line broken over a gap of more than 30 min. Prints the path of each PNG.
Gives values, not a verdict. Run from the root of the repository.
"""
import argparse
import bisect
import json
import os
import statistics
import sys
from datetime import timedelta
import data
from common import QUALITY as METRICS, duration, group, per_step, sensors
HIGH_BAD = ("fwhm", "dominance") # for these two, it is the rise that says a degraded image
COLOURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red",
"Displacement": "tab:purple", "Perturbation": "tab:cyan", "Target issue": "tab:brown", "No data": "black"}
def windows(text):
"""'T1/T2,T3/T4' -> [(start, end)]."""
return [tuple(data.parse_time(b) for b in f.split("/")) for f in text.split(",")] if text else []
def within(t, wins):
return any(a <= t <= b for a, b in wins)
def references(by_sensor, calm):
"""{sensor: {metric: {hour: median, None: median over all hours}}} over the calm measurements."""
ref = {}
for s, ms in by_sensor.items():
ref[s] = {}
for c in METRICS:
per_hour, every = {}, []
for m in ms:
if m[c] is None or (calm and not within(m["t"], calm)):
continue
per_hour.setdefault(m["t"].hour, []).append(m[c])
every.append(m[c])
r = {h: statistics.median(v) for h, v in per_hour.items()}
r[None] = statistics.median(every) if every else None
ref[s][c] = r
return ref
def ratio(ref, s, c, m):
r = ref[s][c].get(m["t"].hour, ref[s][c][None])
return m[c] / r if m[c] is not None and r else None
def affected(r, c, threshold):
return r is not None and ((c in ("flux", "snr") and r < threshold) or (c == "dominance" and r > 1 / threshold))
def hit(ref, s, m, threshold):
"""The measurement says a degraded image: flux or snr below threshold, or dominance above 1/threshold."""
return any(affected(ratio(ref, s, c, m), c, threshold) for c in ("flux", "snr", "dominance"))
def med(v):
return f"{statistics.median(v):.2f}" if v else "-"
def summary(by_sensor, ref, names, refs, pmax):
print(f"maximum peak of the folder: {pmax if pmax is None else f'{pmax:g}'}; pk/max: median peak of the sensor over the "
"window, divided by this maximum")
print("Summary: median ratio to the calm reference, then the extreme and its time (UTC)")
print(f"{'sensor':>12} {'rej':>4} {'pk/max':>6} " + " ".join(f"{c:>26}" for c in METRICS))
meds = {c: {"targets": [], "references": []} for c in METRICS}
for s in names:
ms = by_sensor[s]
if not ms:
continue
cells = []
for c in METRICS:
rs = [(ratio(ref, s, c, m), m["t"]) for m in ms]
rs = [x for x in rs if x[0] is not None]
if not rs:
cells.append(f"{'empty':>26}")
continue
md = statistics.median(r for r, _ in rs)
ext = (max if c in HIGH_BAD else min)(rs, key=lambda x: x[0])
meds[c]["references" if s in refs else "targets"].append(md)
cells.append(f"{md:6.2f} ({ext[0]:6.2f} {ext[1]:%d/%m %H:%M})")
rej = sum(1 for m in ms if data.rejected(m))
pk = [m["peak"] for m in ms if m["peak"] is not None]
pk = f"{statistics.median(pk) / pmax:6.2f}" if pk and pmax else f"{'-':>6}"
print(f"{s + ('r' if s in refs else ''):>12} {rej:>4} {pk} " + " ".join(cells))
for g in ("targets", "references"):
print(f"{g:>24} " + " ".join(f"{statistics.median(meds[c][g]):26.2f}" if meds[c][g] else f"{'-':>26}"
for c in METRICS))
def table(by_sensor, ref, names, c, step, threshold):
key, cells = per_step(min(m["t"] for s in names for m in by_sensor[s]), duration(step)), {}
for s in names:
for m in by_sensor[s]:
r = ratio(ref, s, c, m)
if r is not None:
cells.setdefault(key(m["t"]), {}).setdefault(s, []).append((r, data.rejected(m)))
high = c in HIGH_BAD
print(f"\nTable of {c} / reference, step of {step} ('*': a measurement of the step rejected)")
w = max(7, *map(len, names))
header = f"{'time':16} " + " ".join(f"{s:>{w}}" for s in names) + f" {'med':>6} {'n' + ('>' if high else '<') + str(threshold):>6}"
for i, k in enumerate(sorted(cells)):
if i % 24 == 0:
print(header)
line, vals = [], []
for s in names:
v = cells[k].get(s)
if not v:
line.append(f"{'':>{w}}")
continue
r = statistics.median(x for x, _ in v)
vals.append(r)
line.append(f"{r:{w - 1}.2f}" + ("*" if any(j for _, j in v) else " "))
n = sum(1 for r in vals if (r > 1 / threshold if high else r < threshold))
print(f"{k:%Y-%m-%d %H:%M} " + " ".join(line) + f" {statistics.median(vals):6.2f} {n:>6}")
def common(by_sensor, ref, names, refs, step):
"""Per step: median of the ratios of the targets, then of the references, for each metric."""
key, cells = per_step(min(m["t"] for s in names for m in by_sensor[s]), duration(step)), {}
for s in names:
for m in by_sensor[s]:
d = cells.setdefault(key(m["t"]), {c: ([], []) for c in METRICS})
for c in METRICS:
r = ratio(ref, s, c, m)
if r is not None:
d[c][s in refs].append(r)
print(f"\nMedians of the ratios per step of {step}: targets / references")
header = f"{'time':16} " + " ".join(f"{c:>13}" for c in METRICS)
for i, k in enumerate(sorted(cells)):
if i % 24 == 0:
print(header)
print(f"{k:%Y-%m-%d %H:%M} " + " ".join(
f"{statistics.median(cells[k][c][0]) if cells[k][c][0] else float('nan'):6.2f}/"
f"{statistics.median(cells[k][c][1]) if cells[k][c][1] else float('nan'):<6.2f}" for c in METRICS))
def read_labels(path):
return [json.loads(line) for line in open(path) if line.strip()]
def labels(a, every, ref, refs, names, start, end):
margin = timedelta(minutes=a.margin)
times = {s: [m["t"] for m in every[s]] for s in names}
keep = {int(n) for n in a.numbers.split(",")} if a.numbers else None
print(f"Calm reference: {a.calm or 'the whole folder'}; median ratios before|range|after ({a.margin} min)")
for n, e in enumerate(read_labels(a.labels), 1):
t0, t1 = data.parse_time(e["start"]), data.parse_time(e["end"])
if (keep and n not in keep) or (start and t0 < start) or (end and t0 > end):
continue
print(f"\n{n}. {e['start']} → {e['end']}, {e['family']}")
win = {s: every[s][bisect.bisect_left(times[s], t0 - margin):bisect.bisect_right(times[s], t1 + margin)] for s in names}
parts = [(t0 - margin, t0 - timedelta(seconds=1)), (t0, t1), (t1 + timedelta(seconds=1), t1 + margin)]
for c in METRICS:
cells = []
for g in (False, True):
vals = []
for a0, a1 in parts:
v = [ratio(ref, s, c, m) for s in win if (s in refs) == g for m in win[s] if a0 <= m["t"] <= a1]
vals.append(med([x for x in v if x is not None]))
cells.append(("ref " if g else "tgt ") + "|".join(vals))
print(f" {c:>9} " + " ".join(cells))
worst = []
for s in names:
ms = [m for m in win[s] if t0 <= m["t"] <= t1]
if any(hit(ref, s, m, a.threshold) for m in ms):
fl = [x for x in (ratio(ref, s, "flux", m) for m in ms) if x is not None]
worst.append((min(fl) if fl else 1, s + ("r" if s in refs else "")))
worst.sort()
print(f" affected ({len(worst)}): " + ", ".join(f"{s} {f:.2f}" for f, s in worst) if worst else " affected: none")
if a.edges:
inst = {}
for s in win:
for m in win[s]:
d = inst.setdefault(m["t"], {"n": 0, "flux": [], "snr": [], "dominance": []})
rs = {c: ratio(ref, s, c, m) for c in ("flux", "snr", "dominance")}
d["n"] += any(affected(rs[c], c, a.threshold) for c in rs)
if s not in refs:
for c in rs:
if rs[c] is not None:
d[c].append(rs[c])
print(" time n flux snr dom")
for t in sorted(inst):
d = inst[t]
print(f" {t:%Y-%m-%d %H:%M} {d['n']:>3} {med(d['flux']):>5} {med(d['snr']):>5} {med(d['dominance']):>5}"
+ (" <" if t0 <= t <= t1 else ""))
def outside(a, every, ref, refs, names, start, end):
excluded = set(a.except_.split(",")) if a.except_ else set()
covered = [(data.parse_time(e["start"]), data.parse_time(e["end"])) for e in read_labels(a.labels)
if e["family"] not in excluded]
inst = {}
for s in names:
for m in data.filter_window(every[s], start, end):
if within(m["t"], covered):
continue
if hit(ref, s, m, a.threshold):
inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = ratio(ref, s, "flux", m) or 0
runs = group(sorted(t for t in inst if len(inst[t]) >= a.min), lambda x, y: y - x <= timedelta(minutes=15))
print(f"Outside the labels (except {a.except_}): times when at least {a.min} sensors are affected; {len(runs)} runs")
for ru in runs:
worst = {}
for t in ru:
for s, f in inst[t].items():
worst[s] = min(worst.get(s, 9), f)
print(f"{ru[0]:%Y-%m-%d %H:%M} → {ru[-1]:%H:%M} ({len(ru)}): "
+ ", ".join(f"{s} {f:.2f}" for s, f in sorted(worst.items(), key=lambda x: x[1])))
def pieces(pts, gap=timedelta(minutes=30)):
"""[(t, ...)] -> lists broken where two times are more than `gap` apart: a line does not cross a measurement gap."""
out = []
for i, x in enumerate(pts):
if not out or x[0] - pts[i - 1][0] > gap:
out.append([])
out[-1].append(x)
return out
def figure(a, by_sensor, ref, names, refs, start, end):
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
ranges = [(x, y, "grey") for x, y in windows(",".join(a.zone or []))]
if a.labels:
for e in read_labels(a.labels):
x, y = data.parse_time(e["start"]), data.parse_time(e["end"])
if (start is None or y >= start) and (end is None or x <= end):
ranges.append((x, y, COLOURS.get(e["family"], "grey")))
station = os.path.basename(os.path.normpath(a.folder))
t0 = min(m["t"] for s in names for m in by_sensor[s])
t1 = max(m["t"] for s in names for m in by_sensor[s])
suffix = f"{t0:%Y%m%dT%H%MZ}_{t1:%Y%m%dT%H%MZ}"
os.makedirs(a.out, exist_ok=True)
def axis(ax):
for x, y, colour in ranges:
ax.axvspan(x, y, color=colour, alpha=0.12, lw=0)
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%M"))
ax.set_yscale("log")
ax.axhline(1, ls=":", c="k", lw=0.8)
ax.set_xlim(t0, t1)
fig, axes = plt.subplots(len(METRICS), 1, figsize=(16, 2.4 * len(METRICS)), sharex=True)
for ax, c in zip(axes, METRICS):
inst = {}
for s in names:
for m in by_sensor[s]:
r = ratio(ref, s, c, m)
if r is not None and r > 0:
inst.setdefault(m["t"], ([], []))[s in refs].append(r)
ts = sorted(inst)
tgt = [(t, sorted(inst[t][0])) for t in ts if inst[t][0]]
rfs = [(t, statistics.median(inst[t][1])) for t in ts if inst[t][1]]
for i, mc in enumerate(pieces(tgt)):
ax.fill_between([t for t, _ in mc], [v[len(v) // 10] for _, v in mc], [v[(len(v) * 9) // 10] for _, v in mc],
color="0.8", lw=0, step="mid")
ax.plot([t for t, _ in mc], [statistics.median(v) for _, v in mc], c="k", lw=0.7,
label="targets (median, 10-90 % band)" if i == 0 else None)
for i, mc in enumerate(pieces(rfs)):
ax.plot([t for t, _ in mc], [v for _, v in mc], c="tab:red", lw=0.7, label="references (median)" if i == 0 else None)
axis(ax)
ax.set_ylabel(c)
axes[0].legend(loc="upper right", fontsize=8)
axes[0].set_title(f"{station}: quality metrics / calm reference ({a.calm or 'the whole folder'}), same UTC hour")
path = os.path.join(a.out, f"{station}_quality-common_{suffix}.png")
fig.tight_layout()
fig.savefig(path, dpi=90)
plt.close(fig)
print(path)
for c in a.metrics.split(","):
col = 6
nrows = (len(names) + col - 1) // col
fig, axes = plt.subplots(nrows, col, figsize=(3.2 * col, 1.9 * nrows), sharex=True, squeeze=False)
for ax, s in zip(axes.flat, names):
pts = [(m["t"], ratio(ref, s, c, m)) for m in by_sensor[s]]
pts = [(t, r) for t, r in pts if r is not None and r > 0]
for mc in pieces(pts):
ax.plot([t for t, _ in mc], [r for _, r in mc], lw=0.5, c="tab:red" if s in refs else "k")
axis(ax)
ax.set_title(s, fontsize=8, color="tab:red" if s in refs else "k")
ax.tick_params(labelsize=6)
for ax in list(axes.flat)[len(names):]:
ax.axis("off")
fig.suptitle(f"{station}: {c} / calm reference, per sensor (references in red)")
path = os.path.join(a.out, f"{station}_quality-{c}_{suffix}.png")
fig.tight_layout()
fig.savefig(path, dpi=80)
plt.close(fig)
print(path)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("folder")
p.add_argument("--start", type=data.parse_time)
p.add_argument("--end", type=data.parse_time)
p.add_argument("--sensors", help="comma-separated list (default: all)")
p.add_argument("--calm", help="windows of the reference, T1/T2,T3/T4 (default: the whole folder)")
p.add_argument("--common", action="store_true", help="medians of the targets and of the references per step")
p.add_argument("--table", choices=METRICS, help="ratio of one metric per step and per sensor")
p.add_argument("--step", default="15min", type=lambda x: duration(x) and x, help="15min, 1h, 1d")
p.add_argument("--threshold", type=float, default=0.5, help="\"affected\": flux or snr below threshold, dominance above 1/threshold")
p.add_argument("--labels", help="label file (one JSON per line)")
p.add_argument("--numbers", help="numbers of the labels to read, 3,7")
p.add_argument("--margin", type=int, default=60, help="minutes before and after each label")
p.add_argument("--edges", action="store_true")
p.add_argument("--outside", action="store_true", help="affected times that no label covers")
p.add_argument("--min", type=int, default=1)
p.add_argument("--except", dest="except_", default="Thermal displacement", help="families ignored by --outside, A,B")
p.add_argument("--figure", action="store_true", help="PNG (matplotlib: .venv/bin/python)")
p.add_argument("--metrics", default="flux,snr,dominance", help="per-sensor PNGs of --figure")
p.add_argument("--zone", action="append", help="range to shade on the PNGs, T1/T2 (repeatable)")
p.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
a = p.parse_args()
if a.outside and not a.labels:
p.error("--outside requires --labels")
unknown = [c for c in a.metrics.split(",") if c not in METRICS]
if unknown:
p.error(f"unknown metrics: {', '.join(unknown)}; known: {', '.join(METRICS)}")
every, names = sensors(a.folder, a.sensors)
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
"tools/DATA.md).")
missing = [s for s in names if s not in every]
if missing:
sys.exit(f"Missing sensors: {', '.join(missing)}. Present: {', '.join(sorted(every))}.")
ref = references(every, windows(a.calm))
refs = {s for s in every if data.is_reference(every[s])}
by_sensor = {s: data.filter_window(every[s], a.start, a.end) for s in names}
if not any(by_sensor.values()):
sys.exit("No measurement in the requested window.")
if a.figure:
return figure(a, by_sensor, ref, names, refs, a.start, a.end)
if a.labels:
return (outside if a.outside else labels)(a, every, ref, refs, names, a.start, a.end)
pmax = max((m["peak"] for ms in every.values() for m in ms if m["peak"] is not None), default=None)
print(f"Calm reference: {a.calm or 'the whole folder'}, per UTC hour of the day (r = reference)")
summary(by_sensor, ref, names, refs, pmax)
if a.common:
common(by_sensor, ref, names, refs, a.step)
if a.table:
table(by_sensor, ref, names, a.table, a.step, a.threshold)
def self_test():
"""Check of the ratio to the reference: same hour, fallback on all hours, calm window respected."""
from datetime import datetime, timezone
t = lambda j, h: datetime(2026, 10, j, h, tzinfo=timezone.utc) # noqa: E731
empty = dict.fromkeys(METRICS)
by_sensor = {"A": [dict(empty, t=t(1, 3), flux=10.0), dict(empty, t=t(2, 3), flux=30.0), dict(empty, t=t(2, 5), flux=100.0),
dict(empty, t=t(9, 3), flux=1000.0)]}
ref = references(by_sensor, [(t(1, 0), t(3, 0))])
assert ref["A"]["flux"][3] == 20.0 and ref["A"]["flux"][None] == 30.0, ref
assert ratio(ref, "A", "flux", dict(empty, t=t(9, 3), flux=10.0)) == 0.5
assert ratio(ref, "A", "flux", dict(empty, t=t(9, 7), flux=60.0)) == 2.0 # hour without calm: all hours
assert ratio(ref, "A", "snr", dict(empty, t=t(9, 3), snr=5.0)) is None
assert affected(0.4, "flux", 0.5) and affected(2.5, "dominance", 0.5) and not affected(2.5, "flux", 0.5)
print("self-test: ok")
if __name__ == "__main__":
self_test() if sys.argv[1:] == ["--self-test"] else main()