Regression computed by bench/compare.py (Total > 1 point; station > 5 points and > 20 instants), complexity in the benchmark, pruning after a fix
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
63d954a126
commit
a008ea8fcc
4 files changed
+77
-5
No files matched your search
@@ -0,0 +1,61 @@
|
||||
"""Two benchmark outputs (regression.py), before and after: what changed, and the regressions by the definition of
|
||||
CLAUDE.md, computed.
|
||||
- A family Total: "found" or "correct" loses more than TOTAL_POINTS.
|
||||
- A family of a station: "found" or "correct" loses more than POINTS and more than INSTANTS instants ("found": labelled
|
||||
instants no longer flagged; "correct": wrongly flagged instants added). Below, the loss is noise of a few instants.
|
||||
Then the complexity before → after. The verdict is the reader's: a drop the labels explain (CLAUDE.md, "Not a
|
||||
regression") is said, not hidden.
|
||||
Usage: python3 compare.py <before.txt> <after.txt>
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
TOTAL_POINTS = 1
|
||||
POINTS = 5
|
||||
INSTANTS = 20
|
||||
CELL = r"(\d+)/(\d+) \(\d+ %\)|—"
|
||||
|
||||
|
||||
def parse(path):
|
||||
"""Benchmark output -> ({(station, family): (both, labelled, flagged)}, complexity line or '')."""
|
||||
out, station, cx = {}, None, ""
|
||||
for line in open(path, encoding="utf-8"):
|
||||
if line.startswith("Complexity:"):
|
||||
cx = line.split(":", 1)[1].strip()
|
||||
elif line.strip() and not line.startswith(" "):
|
||||
station = line.split(":")[0].strip()
|
||||
elif m := re.match(rf" (\S.*?)\s{{2,}}({CELL})\s+({CELL})\s*$", line):
|
||||
g = m.groups() # family, found (both, labelled), correct (both, flagged)
|
||||
out[(station, g[0])] = (int(g[2] or g[5] or 0), int(g[3] or 0), int(g[6] or 0))
|
||||
return out, cx
|
||||
|
||||
|
||||
def pct(a, b):
|
||||
return 100 * a / b if b else None
|
||||
|
||||
|
||||
def changes(before, after):
|
||||
"""-> [(station, family, measure, before %, after %, lost instants, regression)] for every measure that moved."""
|
||||
rows = []
|
||||
for key in sorted(set(before) | set(after)):
|
||||
(b0, l0, f0), (b1, l1, f1) = before.get(key, (0, 0, 0)), after.get(key, (0, 0, 0))
|
||||
for measure, p0, p1, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1),
|
||||
("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))):
|
||||
if p0 is None or p1 is None or round(p0) == round(p1):
|
||||
continue
|
||||
drop = p0 - p1
|
||||
bad = drop > TOTAL_POINTS if key[0] == "Total" else drop > POINTS and lost > INSTANTS
|
||||
rows.append((*key, measure, p0, p1, lost, bad))
|
||||
return rows
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit(__doc__)
|
||||
(before, cx0), (after, cx1) = parse(sys.argv[1]), parse(sys.argv[2])
|
||||
rows = changes(before, after)
|
||||
for station, family, measure, p0, p1, lost, bad in rows:
|
||||
print(f"{'REGRESSION' if bad else '':<11}{station:<13}{family:<22}{measure:<8}{p0:4.0f} → {p1:3.0f} %"
|
||||
+ (f" ({lost} instants lost)" if p1 < p0 else ""))
|
||||
n = sum(r[-1] for r in rows)
|
||||
print(f"\n{n} regression(s); complexity: {cx0 or '?'} → {cx1 or '?'}")
|
||||
Reference in new issue
Block a user