fix(benchmarks): count unfenced code, ASCII-safe output, refresh llama3.2 results (#67)

Fixes the local benchmark LOC counter (counted only fenced code, scored bare output 0), makes summary output ASCII-safe (a Unicode arrow crashed the script on Windows cp1252), gitignores generated artifacts, and refreshes the llama3.2 writeup with n=5 data showing the LOC effect is within the noise floor. Follow-up to #63. Verified live.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
DietrichGebert
2026-06-15 16:22:32 +02:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 386f95734a
commit 2e6a93765a
4 changed files with 50 additions and 36 deletions
+5 -4
View File
@@ -37,9 +37,10 @@ def load_arms():
def count_loc(text):
"""Non-blank, non-comment lines inside fenced code blocks."""
"""Non-blank, non-comment lines of code: fenced blocks, or the whole
response when the model emitted bare code with no fence."""
blocks = re.findall(r"```[a-zA-Z0-9_+\-]*\n([\s\S]*?)```", text)
lines = "\n".join(blocks).splitlines()
lines = ("\n".join(blocks) if blocks else text).splitlines()
return sum(
1 for l in lines
if l.strip()
@@ -110,7 +111,7 @@ def run(model, repeat, ollama_url):
sep = "-" * len(header)
print(f"\n{'=' * 60}")
print(f" RESULTS {model} (n={repeat}, median)")
print(f" RESULTS - {model} (n={repeat}, median)")
print(f"{'=' * 60}")
print(f"\nCode LOC per task (median)")
@@ -139,7 +140,7 @@ def run(model, repeat, ollama_url):
out = Path(__file__).parent / "benchmark-local-results.json"
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
print(f"\nFull responses {out}")
print(f"\nFull responses -> {out}")
def main():