Files
aimsight-detectors/bench/compare.py
T

62 lines
2.7 KiB
Python

"""Two benchmark outputs (regression.py), before and after: what changed, and the regressions by the definition of
CLAUDE.md, computed.
- A family Total: "found" or "correct" loses more than TOTAL_POINTS.
- A family of a station: "found" or "correct" loses more than POINTS and more than INSTANTS instants ("found": labelled
instants no longer flagged; "correct": wrongly flagged instants added). Below, the loss is noise of a few instants.
Then the complexity before → after. The verdict is the reader's: a drop the labels explain (CLAUDE.md, "Not a
regression") is said, not hidden.
Usage: python3 compare.py <before.txt> <after.txt>
"""
import re
import sys
TOTAL_POINTS = 1
POINTS = 5
INSTANTS = 20
CELL = r"(\d+)/(\d+) \(\d+ %\)|—"
def parse(path):
"""Benchmark output -> ({(station, family): (both, labelled, flagged)}, complexity line or '')."""
out, station, cx = {}, None, ""
for line in open(path, encoding="utf-8"):
if line.startswith("Complexity:"):
cx = line.split(":", 1)[1].strip()
elif line.strip() and not line.startswith(" "):
station = line.split(":")[0].strip()
elif m := re.match(rf" (\S.*?)\s{{2,}}({CELL})\s+({CELL})\s*$", line):
g = m.groups() # family, found (both, labelled), correct (both, flagged)
out[(station, g[0])] = (int(g[2] or g[5] or 0), int(g[3] or 0), int(g[6] or 0))
return out, cx
def pct(a, b):
return 100 * a / b if b else None
def changes(before, after):
"""-> [(station, family, measure, before %, after %, lost instants, regression)] for every measure that moved."""
rows = []
for key in sorted(set(before) | set(after)):
(b0, l0, f0), (b1, l1, f1) = before.get(key, (0, 0, 0)), after.get(key, (0, 0, 0))
for measure, p0, p1, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1),
("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))):
if p0 is None or p1 is None or round(p0) == round(p1):
continue
drop = p0 - p1
bad = drop > TOTAL_POINTS if key[0] == "Total" else drop > POINTS and lost > INSTANTS
rows.append((*key, measure, p0, p1, lost, bad))
return rows
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.exit(__doc__)
(before, cx0), (after, cx1) = parse(sys.argv[1]), parse(sys.argv[2])
rows = changes(before, after)
for station, family, measure, p0, p1, lost, bad in rows:
print(f"{'REGRESSION' if bad else '':<11}{station:<13}{family:<22}{measure:<8}{p0:4.0f} → {p1:3.0f} %"
+ (f" ({lost} instants lost)" if p1 < p0 else ""))
n = sum(r[-1] for r in rows)
print(f"\n{n} regression(s); complexity: {cx0 or '?'} → {cx1 or '?'}")