"""Two benchmark outputs (regression.py), before and after: what changed, and the regressions by the definition of CLAUDE.md, computed. - A family Total: "found" or "correct" loses more than TOTAL_POINTS. - A family of a station: "found" or "correct" loses more than POINTS and more than INSTANTS instants ("found": labelled instants no longer flagged; "correct": wrongly flagged instants added). Below, the loss is noise of a few instants. Then the complexity before → after. The verdict is the reader's: a drop the labels explain (CLAUDE.md, "Not a regression") is said, not hidden. Usage: python3 compare.py """ import re import sys TOTAL_POINTS = 1 POINTS = 5 INSTANTS = 20 CELL = r"(\d+)/(\d+) \(\d+ %\)|—" def parse(path): """Benchmark output -> ({(station, family): (both, labelled, flagged)}, complexity line or '').""" out, station, cx = {}, None, "" for line in open(path, encoding="utf-8"): if line.startswith("Complexity:"): cx = line.split(":", 1)[1].strip() elif line.strip() and not line.startswith(" "): station = line.split(":")[0].strip() elif m := re.match(rf" (\S.*?)\s{{2,}}({CELL})\s+({CELL})\s*$", line): g = m.groups() # family, found (both, labelled), correct (both, flagged) out[(station, g[0])] = (int(g[2] or g[5] or 0), int(g[3] or 0), int(g[6] or 0)) return out, cx def pct(a, b): return 100 * a / b if b else None def changes(before, after): """-> [(station, family, measure, before %, after %, lost instants, regression)] for every measure that moved.""" rows = [] for key in sorted(set(before) | set(after)): (b0, l0, f0), (b1, l1, f1) = before.get(key, (0, 0, 0)), after.get(key, (0, 0, 0)) for measure, p0, p1, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1), ("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))): if p0 is None or p1 is None or round(p0) == round(p1): continue drop = p0 - p1 bad = drop > TOTAL_POINTS if key[0] == "Total" else drop > POINTS and lost > INSTANTS rows.append((*key, measure, p0, p1, lost, bad)) return rows if __name__ == "__main__": if len(sys.argv) != 3: sys.exit(__doc__) (before, cx0), (after, cx1) = parse(sys.argv[1]), parse(sys.argv[2]) rows = changes(before, after) for station, family, measure, p0, p1, lost, bad in rows: print(f"{'REGRESSION' if bad else '':<11}{station:<13}{family:<22}{measure:<8}{p0:4.0f} → {p1:3.0f} %" + (f" ({lost} instants lost)" if p1 < p0 else "")) n = sum(r[-1] for r in rows) print(f"\n{n} regression(s); complexity: {cx0 or '?'} → {cx1 or '?'}")