From 2e6a93765aabb345b7325cdcac4c8efef7955644 Mon Sep 17 00:00:00 2001 From: DietrichGebert Date: Mon, 15 Jun 2026 16:22:32 +0200 Subject: [PATCH] fix(benchmarks): count unfenced code, ASCII-safe output, refresh llama3.2 results (#67) Fixes the local benchmark LOC counter (counted only fenced code, scored bare output 0), makes summary output ASCII-safe (a Unicode arrow crashed the script on Windows cp1252), gitignores generated artifacts, and refreshes the llama3.2 writeup with n=5 data showing the LOC effect is within the noise floor. Follow-up to #63. Verified live. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 4 ++ benchmarks/benchmark-local.py | 9 +-- benchmarks/loc.js | 5 +- .../results/2026-06-15-llama3.2-local.md | 68 +++++++++++-------- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 7188f92..3f3335b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ node_modules/ # promptfoo eval artifacts .promptfoo/ benchmarks/output* +benchmarks/benchmark-local-results.json + +# Python +__pycache__/ # one-off social/announcement art, not repo content announce-*.png diff --git a/benchmarks/benchmark-local.py b/benchmarks/benchmark-local.py index c3520ec..7e9a2d2 100644 --- a/benchmarks/benchmark-local.py +++ b/benchmarks/benchmark-local.py @@ -37,9 +37,10 @@ def load_arms(): def count_loc(text): - """Non-blank, non-comment lines inside fenced code blocks.""" + """Non-blank, non-comment lines of code: fenced blocks, or the whole + response when the model emitted bare code with no fence.""" blocks = re.findall(r"```[a-zA-Z0-9_+\-]*\n([\s\S]*?)```", text) - lines = "\n".join(blocks).splitlines() + lines = ("\n".join(blocks) if blocks else text).splitlines() return sum( 1 for l in lines if l.strip() @@ -110,7 +111,7 @@ def run(model, repeat, ollama_url): sep = "-" * len(header) print(f"\n{'=' * 60}") - print(f" RESULTS — {model} (n={repeat}, median)") + print(f" RESULTS - {model} (n={repeat}, median)") print(f"{'=' * 60}") print(f"\nCode LOC per task (median)") @@ -139,7 +140,7 @@ def run(model, repeat, ollama_url): out = Path(__file__).parent / "benchmark-local-results.json" out.write_text(json.dumps(results, indent=2), encoding="utf-8") - print(f"\nFull responses → {out}") + print(f"\nFull responses -> {out}") def main(): diff --git a/benchmarks/loc.js b/benchmarks/loc.js index b088fbb..3640fa0 100644 --- a/benchmarks/loc.js +++ b/benchmarks/loc.js @@ -1,9 +1,10 @@ -// Deterministic code-size metric: non-blank, non-comment lines inside fenced code blocks. +// Deterministic code-size metric: non-blank, non-comment lines of code. Counts +// fenced blocks, or the whole response when the model emitted bare code unfenced. // Recorded as the `code_loc` metric per arm (always passes; it is a measurement, not a gate). module.exports = (output) => { const text = String(output || ''); const blocks = [...text.matchAll(/```[a-zA-Z0-9_+-]*\n([\s\S]*?)```/g)].map((m) => m[1]); - const code = blocks.join('\n'); + const code = blocks.length ? blocks.join('\n') : text; const loc = code .split('\n') .map((l) => l.trim()) diff --git a/benchmarks/results/2026-06-15-llama3.2-local.md b/benchmarks/results/2026-06-15-llama3.2-local.md index 857a169..55ed4e0 100644 --- a/benchmarks/results/2026-06-15-llama3.2-local.md +++ b/benchmarks/results/2026-06-15-llama3.2-local.md @@ -2,48 +2,52 @@ Same 5 tasks as the Claude benchmark, same three arms (baseline / caveman / ponytail), run against a local **llama3.2:latest** (3.2B, Q4_K_M) via Ollama on a Windows 11 machine. -n=1 per cell. Tooling: `benchmarks/benchmark-local.py` (no promptfoo needed). +Tooling: `benchmarks/benchmark-local.py` (no promptfoo needed). -## Results +> **Updated 2026-06-15:** the LOC counter now counts bare, unfenced code. It +> previously counted only fenced code blocks and scored everything else as 0, +> which silently deflated any arm whose output happened to skip the fences (small +> models do this often). Numbers below use the corrected counter at n=5 median. +> Absolute times reflect this machine (GPU-accelerated); compare arms within a +> run, not against an earlier CPU-bound machine. + +## Results (n=5, median) **Code LOC** | arm | email | debounce | csv-sum | countdown | rate-limit | **TOTAL** | |---|--:|--:|--:|--:|--:|--:| -| baseline | 13 | 13 | 38 | 44 | 32 | **140** | -| caveman | 16 | 12 | 5 | 45 | 28 | **106** | -| ponytail | 18 | 21 | 7 | 38 | 49 | **133** | +| baseline | 16 | 18 | 22 | 37 | 16 | **109** | +| caveman | 16 | 21 | 18 | 46 | 32 | **133** | +| ponytail | 17 | 22 | 18 | 52 | 28 | **137** | **Time (seconds)** | arm | email | debounce | csv-sum | countdown | rate-limit | **TOTAL** | |---|--:|--:|--:|--:|--:|--:| -| baseline | 52.6 | 37.2 | 63.4 | 65.2 | 71.0 | **289.4** | -| caveman | 79.6 | 54.4 | 28.3 | 71.4 | 61.0 | **294.7** | -| ponytail | 99.7 | 71.0 | 25.4 | 74.8 | 97.3 | **368.2** | - -**LOC vs baseline** - -| arm | total LOC | vs baseline | -|---|--:|---| -| caveman | 106 | −24% | -| ponytail | 133 | −5% | +| baseline | 3.1 | 3.7 | 3.6 | 4.2 | 4.8 | **19.4** | +| caveman | 4.1 | 4.2 | 3.6 | 4.4 | 4.8 | **21.1** | +| ponytail | 4.1 | 4.2 | 3.8 | 4.8 | 4.9 | **21.8** | ## Key findings -**Ponytail does not transfer to llama3.2.** On 3 of 5 tasks (email: 13→18, debounce: 13→21, -rate-limit: 32→49) ponytail produced *more* code than the no-skill baseline. Total LOC -reduction was −5% vs the 80–94% seen on Claude. Response time increased by +27% (368s vs -289s) rather than the 3–6× speedup seen on Claude. +**On llama3.2 the LOC effect is inside the noise floor.** At temperature 0.7 the +per-run totals swing hard: across the five runs, ponytail landed anywhere from +17% *below* baseline to 50% *above* it. The n=5 median came out +26%; a separate +n=3 median came out −17%. The aggregate itself flips sign depending on the +sample, and the countdown task alone ranged 19 to 74 LOC on baseline. There is no +stable LOC reduction to report. -**Caveman outperformed ponytail on this model** (−24% LOC, similar time to baseline). -Caveman's rules are simpler prose instructions that a small model can follow more reliably; -ponytail's multi-step decision ladder requires a stronger instruction-follower. +**Ponytail does not transfer to llama3.2.** The 80-94% LOC reduction seen on +Claude is simply absent: the signal is lost in run-to-run variance. The one +consistent effect is on time, and it goes the wrong way: ponytail is ~10-15% +*slower* than baseline (more system-prompt tokens to process), never the 3-6x +speedup seen on Claude. -**Why this happens:** Ponytail is a prompt-engineering skill calibrated on Claude models, -which are specifically trained to follow detailed system instructions. A 3.2B quantised model -partially absorbs the ponytail rules and then adds extra prose *justifying* its choices — -paying the complexity cost without getting the minimalism benefit. +**Why:** ponytail is a prompt-engineering skill calibrated on Claude models, +which are trained to follow detailed system instructions. A 3.2B quantised model +absorbs the rules only partially and adds prose justifying its choices, paying +the instruction-following cost without reliably converting it into less code. ## Reproduce @@ -51,9 +55,12 @@ Install Ollama and pull a model, then run from the repo root: ```bash ollama pull llama3.2 -python benchmarks/benchmark-local.py --model llama3.2 +python benchmarks/benchmark-local.py --model llama3.2 --repeat 5 ``` +At this model size the LOC signal is noisy; raise `--repeat` (or lower the +sampling temperature in the script) before reading anything into the totals. + Optional flags: ``` @@ -63,6 +70,7 @@ Optional flags: ## Takeaway -The benchmark claims in the README are accurate for the models tested (Haiku, Sonnet, Opus). -For local/small models, expect significantly smaller — or even negative — gains until -instruction-following capability reaches a threshold comparable to Claude Haiku or better. +The benchmark claims in the README are accurate for the models tested (Haiku, +Sonnet, Opus). For local/small models, expect the gains to shrink into the noise +until instruction-following reaches a threshold comparable to Claude Haiku or +better.