wangjunxiao/ponytail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: comprehension-first guard + reuse rung (#245, #217) (#253)

Browse Source 
#245 "Dangerously lazy": add an operational "fix the root cause, not the
symptom" directive — grep every caller of the function you touch and fix the
shared function once (the smaller diff). Validated on the agentic benchmark: on
a shared-helper bug-fix trap, baseline fixes the root cause 1/6 while ponytail
does 6/6 on both Sonnet 4.6 (the model the issue was filed on) and Opus 4.8,
verified by reading the produced code. Plain prose ("trace the flow") did not
move it; the actionable, lazy-framed directive did.

#217 "Missing rung": add ladder rung 2 "Already in this codebase? Reuse it,
don't re-write it." Propagated across SKILL.md, AGENTS.md, all agent mirror
copies, the hook fallback, and both READMEs (check-rule-copies passes).

Benchmark: 4 new deterministic quality-tier tasks (reuse-slug, reuse-money,
trace-transfer, trace-amount) with selftest-proven good/bad refs; harness gains
multi-file seed support in --selftest, distinctive-behaviour reuse detection,
and counts in-file __main__/demo() self-checks as test LOC (not source bloat)
for surgical tasks. Full writeup in
benchmarks/results/2026-06-22-issue-245-217-comprehension.md.

Also carries the in-progress todo-null benchmark task already present in the
working tree.

Co-authored-by: Dietrich Gebert <dgebert@Dietrichs-MacBook-Pro.local>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DietrichGebert
2026-06-22 23:30:05 +02:00
committed by GitHub
co-authored by Dietrich Gebert Claude Opus 4.8
parent 6da37bfa7d
commit dedc97ca7c
15 changed files with 698 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/agentic/run.py
+46 -5
View File
@@ -89,11 +89,40 @@ def _count(p: Path, with_comments: bool):
n += 1
return n
def code_stats(workdir: Path):
_SELFCHECK_DEFS = ("def demo(", "def _demo(", "def selfcheck(", "def _selfcheck(",
"def _check(", "def _smoke(", "def smoke(")
def _selfcheck_split(p: Path):
"""Split a produced .py file at the first TOP-LEVEL self-check marker (a `__main__` guard or a
demo()/selfcheck() function) through end of file. Returns (src_total, src_code, sc_total,
sc_code), counted like _count. On a surgical task that delivers ONE function, an in-file self-
check is the runnable check ponytail's rule asks for -- a positive signal, not source bloat --
so it is split off here and counted as test LOC instead of penalising the arm that wrote it."""
try: lines = p.read_text(encoding="utf-8", errors="ignore").splitlines()
except Exception: return 0, 0, 0, 0
start = None
for i, ln in enumerate(lines):
if ln[:1] not in (" ", "\t") and (ln.startswith("if __name__") or ln.startswith(_SELFCHECK_DEFS)):
start = i; break
def cnt(seq):
t = c = 0
for ln in seq:
s = ln.strip()
if not s: continue
t += 1
if not s.startswith(("#", "//", "*", "/*", "*/")): c += 1
return t, c
if start is None:
t, c = cnt(lines); return t, c, 0, 0
t, c = cnt(lines[:start]); st, sc = cnt(lines[start:])
return t, c, st, sc
def code_stats(workdir: Path, selfcheck_as_test: bool = False):
"""LOC over code-extension source files only (generated images/data can't pollute it).
total_loc counts every non-blank line including comments and docstrings -- the bloat a vibe
baseline actually produces. src_loc is code-only, for the breakdown. Tests tracked separately,
never as bloat."""
never as bloat. selfcheck_as_test (surgical tasks): an in-file __main__/demo() self-check is
reclassified from source to test, so following ponytail's 'leave a runnable check' rule is not
counted as code bloat against it."""
fixture = set() # files that were seeded, not delivered
fm = workdir / "_fixture_files.json"
if fm.exists():
@@ -105,10 +134,19 @@ def code_stats(workdir: Path):
and not p.name.startswith((".", "_")) and _rel(p) not in fixture]
src = [p for p in files if not _is_test(p, workdir)]
tst = [p for p in files if _is_test(p, workdir)]
test_loc = sum(_count(p, True) for p in tst)
if selfcheck_as_test:
total = code = sc_test = 0
for p in src:
t, c, st, _ = _selfcheck_split(p)
total += t; code += c; sc_test += st
return {"files": len(files), "src_files": len(src),
"total_loc": total, "src_loc": code,
"test_files": len(tst), "test_loc": test_loc + sc_test}
return {"files": len(files), "src_files": len(src),
"total_loc": sum(_count(p, True) for p in src), # incl comments + docstrings (the bloat)
"src_loc": sum(_count(p, False) for p in src), # code only
"test_files": len(tst), "test_loc": sum(_count(p, True) for p in tst)}
"test_files": len(tst), "test_loc": test_loc}
def _git(workdir, *args):
return subprocess.run([shutil.which("git") or "git", *args], cwd=str(workdir),
@@ -151,7 +189,9 @@ def selftest():
axis = task.get("axis", "safe")
for kind in ("good", "bad"):
with tempfile.TemporaryDirectory() as d:
(Path(d) / task["file"]).write_text(task[kind], encoding="utf-8")
for fn, content in task.get("seed", {}).items(): # seed siblings (a helper module
(Path(d) / fn).write_text(content, encoding="utf-8") # the ref imports) too
(Path(d) / task["file"]).write_text(task[kind], encoding="utf-8") # entry = the ref
r = task["score"](Path(d))
ok = (r["correct"] == 1 and r["safe"] == 1) if kind == "good" else (r[axis] == 0)
print(f"{'ok ' if ok else 'XX '} {tid:12} {kind:4} correct={r['correct']} "
@@ -205,7 +245,8 @@ def score_workspace(task_id, arm, model, workdir: Path):
"cache_tokens": (u.get("cache_read_input_tokens") or 0) + (u.get("cache_creation_input_tokens") or 0)}
result_text = j.get("result", "")
except Exception: pass
stats = git_diff_stats(workdir) if TASKS[task_id].get("fixture") else code_stats(workdir)
surgical = not TASKS[task_id].get("open") and not TASKS[task_id].get("fixture")
stats = git_diff_stats(workdir) if TASKS[task_id].get("fixture") else code_stats(workdir, selfcheck_as_test=surgical)
# open/explain tasks answer in the chat, not a file. If no source file was written, count the
# code the agent delivered in its chat answer so the comparison isn't a false zero.
if TASKS[task_id].get("open") and stats["total_loc"] == 0 and result_text: