213d888c90
Re-ran the agent comparison now that MTP-off removed the tool-call
corruption. 4 seeded bug classes x 2 runs x 2 agents, same backend,
same prompts, graders validated (seeded fails / clean passes).
PI 8/8 HERMES 8/8
The earlier 'Pi 3/3 vs Hermes 2/3' was measured on the corrupting
server -- it was the SERVER, not the agent. On a clean server the two
are indistinguishable on these tasks.
Two of MY grader bugs surfaced and were fixed before scoring:
- t2 want_any held 'Number(' / 'String(id)' = invalid regex. Hidden
because parseInt matched first (any() short-circuits); it crashed
only on a fix that avoided parseInt, scoring a PASS as FAIL.
- t3 accepted only one of two valid fixes. generateMysteryImages
declares exec = promisify(execFile) but never calls exec -- dead
code even in the clean original. Deleting it is more minimal than
restoring imports for an unused var. Both agents found that; the
grader called it FAIL.
Uncorrected the bench read 7/8 vs 6/8 -- both were grader artifacts.
44 lines
1.3 KiB
Python
44 lines
1.3 KiB
Python
"""Grade a task workdir. Usage: grade.py <task> <dir> -> prints PASS/FAIL."""
|
|
import json, re, sys
|
|
from pathlib import Path
|
|
|
|
G = json.loads(Path("/tmp/agentbench/graders.json").read_text(encoding="utf-8"))
|
|
|
|
def grade(task, d):
|
|
g = G[task]
|
|
f = Path(d) / g["file"]
|
|
if not f.exists():
|
|
return False, "file missing"
|
|
t = f.read_text(encoding="utf-8")
|
|
|
|
if g.get("forbid") and re.search(g["forbid"], t):
|
|
return False, f"buggy pattern still present: {g['forbid']}"
|
|
|
|
if "want_count" in g:
|
|
pat, n = g["want_count"]
|
|
got = len(re.findall(pat, t))
|
|
if got != n:
|
|
return False, f"expected {n}x /{pat}/, got {got}"
|
|
|
|
if "want_counts" in g:
|
|
for pat, n in g["want_counts"]:
|
|
got = len(re.findall(pat, t))
|
|
if got != n:
|
|
return False, f"expected {n}x /{pat}/, got {got}"
|
|
|
|
if "want_all" in g:
|
|
for pat in g["want_all"]:
|
|
if not re.search(pat, t):
|
|
return False, f"missing required: {pat}"
|
|
|
|
if "want_any" in g:
|
|
if not any(re.search(p, t) for p in g["want_any"]):
|
|
return False, f"none of the accepted fixes present: {g['want_any']}"
|
|
|
|
return True, "ok"
|
|
|
|
if __name__ == "__main__":
|
|
ok, why = grade(sys.argv[1], sys.argv[2])
|
|
print(f"{'PASS' if ok else 'FAIL'} ({why})")
|
|
sys.exit(0 if ok else 1)
|