feat(benchmarks): add correctness assertion (#31)

* feat(benchmarks): add correctness assertion - proves less code is not broken code

The existing benchmark measures lines-of-code (loc.js) but never checks
whether the generated code actually works. This adds a functional
correctness gate (correctness.js) that extracts code from fenced blocks
and runs per-task checks:

- email validator: spawns Python, asserts accept/reject on 5 inputs
- debounce: spawns Node, asserts delayed execution + reset on re-call
- csv sum: spawns Python with a test CSV, asserts correct total (351)
- countdown (React): structural check (useState + useEffect + decrement)
- rate limiter (FastAPI): structural check (limit logic + framework usage)

12 unit tests (node:test) cover good/bad outputs for every task plus the
unknown-task edge case. Existing tests and rule-copy checks unaffected.

* fix: address review feedback

- csv check: use regex lookaround instead of substring match to prevent
  false positives (e.g. 13510 containing '351')
- ratelimit: fix operator precedence in block finder by adding parens
  around the || inside the !b.lang guard
- README: note that React/FastAPI checks are structural only, add
  prerequisites section (Python 3, pandas, Node.js 18+)
- test: add regression test for csv substring false positive

benchmarks/README.md

@@ -40,6 +40,21 @@ Tasks: email validator, JS debounce, CSV sum, React countdown, FastAPI rate-limi
 
 Versus baseline, ponytail writes **80-94% less code**, costs **47-77% less**, and runs **3-6x faster**, on every model.
 
+## Metrics
+
+| File | Metric | Behavior |
+|------|--------|----------|
+| `loc.js` | `loc` | Measurement - always passes, records line count |
+| `correctness.js` | `correct` | Gate - fails if generated code doesn't work |
+
+`correctness.js` extracts fenced code blocks and runs per-task checks (spawns Python/Node for email, debounce, CSV; structural regex for React and FastAPI). A broken one-liner that scores great on LOC will fail on correctness.
+
+> **Note:** The React countdown and FastAPI rate-limit checks are keyword/structural only (no runtime execution), so they verify plausible structure rather than full correctness. The email, debounce, and CSV checks execute the code.
+
+### Prerequisites
+
+Running the benchmark requires **Python 3**, **pandas**, and **Node.js** (18+).
+
 ## Notes
 
 - Caveman is a prose-compression skill (it leaves code "normal"), so it lands between baseline and ponytail on code size and wins mainly on prose tokens.

benchmarks/correctness.js

@@ -0,0 +1,263 @@
+// Functional correctness assertion: runs generated code against lightweight test
+// cases per task. Proves "less code" is not "broken code". Spawns python/node
+// with the extracted code + appended assertions; returns pass/fail + score.
+//
+// Metric: `correct` (1 = all checks pass, 0 = at least one fails).
+// Unlike loc.js (measurement-only), this one is a gate — a wrong answer is a
+// wrong answer regardless of how few lines produced it.
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+// Extract fenced code blocks, tagged by language.
+function extractBlocks(text) {
+  const matches = [...text.matchAll(/```(\w*)\n([\s\S]*?)```/g)];
+  return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] }));
+}
+
+// Identify which task we're evaluating from vars.task.
+function identifyTask(task) {
+  const t = task.toLowerCase();
+  if (t.includes('email') && t.includes('valid')) return 'email';
+  if (t.includes('debounce')) return 'debounce';
+  if (t.includes('csv') && t.includes('sum')) return 'csv';
+  if (t.includes('countdown') && t.includes('react')) return 'countdown';
+  if (t.includes('rate limit') || t.includes('rate-limit')) return 'ratelimit';
+  return null;
+}
+
+// Run a command, return { ok, stderr }.
+function exec(cmd, opts = {}) {
+  try {
+    execSync(cmd, { timeout: 10_000, encoding: 'utf8', stdio: 'pipe', ...opts });
+    return { ok: true, stderr: '' };
+  } catch (e) {
+    return { ok: false, stderr: (e.stderr || e.message || '').slice(0, 500) };
+  }
+}
+
+// Write content to a temp file, return the path.
+function tmpFile(ext, content) {
+  const p = path.join(os.tmpdir(), `ponytail-bench-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`);
+  fs.writeFileSync(p, content);
+  return p;
+}
+
+// --- Per-task test harnesses ---
+
+const CHECKS = {
+  email(blocks) {
+    const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('def ')));
+    if (!code) return { pass: false, reason: 'No Python code block found' };
+
+    // Append assertions that call the generated function by common names.
+    const harness = `
+${code.code}
+
+# Find the validator function
+import sys
+fn = None
+for name in ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate']:
+    if name in dir() and callable(eval(name)):
+        fn = eval(name)
+        break
+
+if fn is None:
+    # Try any function that takes one arg
+    import inspect
+    for name, obj in list(globals().items()):
+        if callable(obj) and not name.startswith('_'):
+            try:
+                sig = inspect.signature(obj)
+                if len(sig.parameters) == 1:
+                    fn = obj
+                    break
+            except (ValueError, TypeError):
+                pass
+
+if fn is None:
+    print("FAIL: no validator function found")
+    sys.exit(1)
+
+# Test cases
+failures = []
+if not fn("user@example.com"):
+    failures.append("rejected valid: user@example.com")
+if not fn("a@b.co"):
+    failures.append("rejected valid: a@b.co")
+if fn("no-at-sign"):
+    failures.append("accepted invalid: no-at-sign")
+if fn(""):
+    failures.append("accepted invalid: empty string")
+if fn("@missing-local.com"):
+    failures.append("accepted invalid: @missing-local.com")
+
+if failures:
+    print("FAIL: " + "; ".join(failures))
+    sys.exit(1)
+print("PASS")
+`;
+    const f = tmpFile('.py', harness);
+    const result = exec(`python "${f}"`);
+    fs.unlinkSync(f);
+    if (result.ok) return { pass: true, reason: 'Email validator passes all checks' };
+    return { pass: false, reason: result.stderr || 'Email validator failed' };
+  },
+
+  debounce(blocks) {
+    const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && b.code.includes('function')));
+    if (!code) return { pass: false, reason: 'No JavaScript code block found' };
+
+    const harness = `
+${code.code}
+
+// Find the debounce function
+const fn = typeof debounce === 'function' ? debounce
+  : typeof module !== 'undefined' && typeof module.exports === 'function' ? module.exports
+  : null;
+
+if (!fn) {
+  console.error("FAIL: no debounce function found");
+  process.exit(1);
+}
+
+// Test: debounced function should not fire immediately
+let callCount = 0;
+const debounced = fn(() => { callCount++; }, 50);
+debounced();
+debounced();
+debounced();
+
+if (callCount > 0) {
+  console.error("FAIL: debounce fired immediately (should wait)");
+  process.exit(1);
+}
+
+// Test: should fire after the delay
+setTimeout(() => {
+  if (callCount !== 1) {
+    console.error("FAIL: expected 1 call after delay, got " + callCount);
+    process.exit(1);
+  }
+  console.log("PASS");
+}, 120);
+`;
+    const f = tmpFile('.mjs', harness);
+    const result = exec(`node "${f}"`);
+    fs.unlinkSync(f);
+    if (result.ok) return { pass: true, reason: 'Debounce passes all checks' };
+    return { pass: false, reason: result.stderr || 'Debounce failed' };
+  },
+
+  csv(blocks) {
+    const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('csv') && b.code.includes('sum')));
+    if (!code) return { pass: false, reason: 'No Python code block found' };
+
+    // Create a test CSV and wrap the generated code so it reads it.
+    const csvContent = 'name,amount\nAlice,100.5\nBob,200.0\nCharlie,50.5\n';
+    const csvPath = tmpFile('.csv', csvContent).replace(/\\/g, '/');
+
+    // The generated code likely reads 'sales.csv'; patch the filename.
+    let patched = code.code.replace(/['"]sales\.csv['"]/g, `'${csvPath}'`);
+    // Also try open() calls
+    patched = patched.replace(/open\(\s*['"]sales\.csv['"]/g, `open('${csvPath}'`);
+
+    const harness = `
+import sys, os
+os.chdir(r"${path.dirname(csvPath)}")
+
+# Capture print output
+import io
+_stdout = sys.stdout
+sys.stdout = io.StringIO()
+
+try:
+${patched.split('\n').map((l) => '    ' + l).join('\n')}
+except Exception as e:
+    sys.stdout = _stdout
+    # If it needs sales.csv in cwd, write it there and retry
+    pass
+
+output = sys.stdout.getvalue()
+sys.stdout = _stdout
+
+# Check output contains the number 351 (100.5 + 200.0 + 50.5)
+# Match as a standalone number (not as substring of e.g. 13510)
+import re
+if re.search(r'(?<![\\d])351(?:\\.0)?(?![\\d])', output):
+    print("PASS")
+else:
+    # Try running it differently: maybe it defines a function
+    print("FAIL: output was: " + repr(output[:200]))
+    sys.exit(1)
+`;
+    const f = tmpFile('.py', harness);
+    const result = exec(`python "${f}"`);
+    try { fs.unlinkSync(f); } catch (e) {}
+    try { fs.unlinkSync(csvPath); } catch (e) {}
+    if (result.ok) return { pass: true, reason: 'CSV sum produces correct result (351)' };
+    return { pass: false, reason: result.stderr || 'CSV sum failed' };
+  },
+
+  countdown(blocks) {
+    // React components can't run in bare Node without a bundler. Structural check:
+    // the code must contain timer/countdown logic (useState/useEffect/setInterval/setTimeout).
+    const code = blocks.find((b) => b.code.includes('ount') || b.code.includes('timer') || b.code.includes('Timer'));
+    if (!code) return { pass: false, reason: 'No countdown component found' };
+
+    const src = code.code;
+    const hasState = /useState|useReducer|this\.state/.test(src);
+    const hasEffect = /useEffect|componentDidMount|setInterval|setTimeout/.test(src);
+    const hasDecrement = /- 1|-= 1|prev - 1|count - 1|seconds - 1|time - 1/.test(src);
+
+    const failures = [];
+    if (!hasState) failures.push('no state management (useState/useReducer)');
+    if (!hasEffect) failures.push('no timer setup (useEffect/setInterval/setTimeout)');
+    if (!hasDecrement) failures.push('no countdown decrement logic');
+
+    if (failures.length === 0) return { pass: true, reason: 'Countdown has required structure' };
+    return { pass: false, reason: 'Missing: ' + failures.join(', ') };
+  },
+
+  ratelimit(blocks) {
+    const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && (b.code.includes('rate') || b.code.includes('limit'))));
+    if (!code) return { pass: false, reason: 'No Python code block found' };
+
+    // Structural check for rate limiting: must have some form of counter/time tracking.
+    const src = code.code;
+    const hasTimeTracking = /time\.|datetime|asyncio/.test(src);
+    const hasLimitLogic = /limit|max_requests|rate|429|Too Many|HTTPException|RateLimiter/.test(src);
+    const hasFastAPI = /fastapi|FastAPI|app\s*=|@app\./.test(src);
+
+    const failures = [];
+    if (!hasLimitLogic) failures.push('no rate limit logic');
+    if (!hasFastAPI) failures.push('no FastAPI usage');
+
+    if (failures.length === 0) return { pass: true, reason: 'Rate limiter has required structure' };
+    return { pass: false, reason: 'Missing: ' + failures.join(', ') };
+  },
+};
+
+// --- Main assertion entry point ---
+
+module.exports = (output, context) => {
+  const task = identifyTask(context.vars.task || '');
+  if (!task) {
+    return { pass: true, score: 1, reason: 'Unknown task, skipped correctness check' };
+  }
+
+  const blocks = extractBlocks(String(output || ''));
+  if (blocks.length === 0) {
+    return { pass: false, score: 0, reason: 'No code blocks in output' };
+  }
+
+  const check = CHECKS[task];
+  const result = check(blocks);
+  return {
+    pass: result.pass,
+    score: result.pass ? 1 : 0,
+    reason: result.reason,
+  };
+};

benchmarks/promptfooconfig.yaml

@@ -29,6 +29,9 @@ defaultTest:
     - type: javascript
       value: file://loc.js
       metric: code_loc
+    - type: javascript
+      value: file://correctness.js
+      metric: correct
 
 tests:
   - vars: { task: "Write me a Python function that validates email addresses." }

tests/correctness.test.js

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+// Unit test for the correctness benchmark assertion. Feeds known-good and
+// known-bad LLM outputs through each task checker and asserts the expected
+// pass/fail verdict. Runs without promptfoo — just node:test + the module.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const correctness = require('../benchmarks/correctness');
+
+// Helper: wrap code in a fenced block and call the assertion with task vars.
+function check(task, lang, code) {
+  const output = '```' + lang + '\n' + code + '\n```';
+  return correctness(output, { vars: { task } });
+}
+
+// --- Email validator ---
+
+test('email: correct one-liner passes', () => {
+  const result = check(
+    'Write me a Python function that validates email addresses.',
+    'python',
+    'def validate_email(email):\n    return "@" in email and "." in email.split("@")[-1] and email.split("@")[0] != ""',
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('email: always-true validator fails', () => {
+  const result = check(
+    'Write me a Python function that validates email addresses.',
+    'python',
+    'def validate_email(email):\n    return True',
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+test('email: no code block fails', () => {
+  const result = correctness('Here is my answer: just use regex.', {
+    vars: { task: 'Write me a Python function that validates email addresses.' },
+  });
+  assert.equal(result.pass, false);
+});
+
+// --- Debounce ---
+
+test('debounce: correct implementation passes', () => {
+  const result = check(
+    'Add debounce to a search input in vanilla JavaScript.',
+    'javascript',
+    `function debounce(fn, delay) {
+  let timer;
+  return function(...args) {
+    clearTimeout(timer);
+    timer = setTimeout(() => fn.apply(this, args), delay);
+  };
+}`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('debounce: immediate-call implementation fails', () => {
+  const result = check(
+    'Add debounce to a search input in vanilla JavaScript.',
+    'javascript',
+    `function debounce(fn, delay) {
+  return function(...args) { fn.apply(this, args); };
+}`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+// --- CSV sum ---
+
+test('csv: correct pandas one-liner passes', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `import pandas as pd
+df = pd.read_csv('sales.csv')
+print(df['amount'].sum())`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('csv: code that prints wrong value fails', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `print(999)`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+test('csv: value containing 351 as substring fails (e.g. 13510)', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `print(13510)`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+// --- React countdown ---
+
+test('countdown: valid React component passes', () => {
+  const result = check(
+    'Build me a countdown timer component in React.',
+    'javascript',
+    `import { useState, useEffect } from 'react';
+export default function Countdown({ seconds }) {
+  const [count, setCount] = useState(seconds);
+  useEffect(() => {
+    if (count <= 0) return;
+    const id = setInterval(() => setCount(prev => prev - 1), 1000);
+    return () => clearInterval(id);
+  }, [count]);
+  return <div>{count}</div>;
+}`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('countdown: static div without state fails', () => {
+  const result = check(
+    'Build me a countdown timer component in React.',
+    'javascript',
+    `export default function Countdown() { return <div>10</div>; }`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+// --- Rate limiter ---
+
+test('ratelimit: FastAPI with limit logic passes', () => {
+  const result = check(
+    'Add rate limiting to my FastAPI endpoint so users can\'t spam it.',
+    'python',
+    `from fastapi import FastAPI, HTTPException
+import time
+
+app = FastAPI()
+requests = {}
+
+@app.get("/api")
+def endpoint(user: str = "anon"):
+    now = time.time()
+    window = requests.get(user, [])
+    window = [t for t in window if now - t < 60]
+    if len(window) >= 10:
+        raise HTTPException(429, "Too Many Requests")
+    window.append(now)
+    requests[user] = window
+    return {"ok": True}`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('ratelimit: plain endpoint without limiting fails', () => {
+  const result = check(
+    'Add rate limiting to my FastAPI endpoint.',
+    'python',
+    `from fastapi import FastAPI
+app = FastAPI()
+
+@app.get("/api")
+def endpoint():
+    return {"ok": True}`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+// --- Edge cases ---
+
+test('unknown task is gracefully skipped', () => {
+  const result = correctness('```python\nprint("hi")\n```', {
+    vars: { task: 'Explain quantum computing.' },
+  });
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+  assert.match(result.reason, /unknown task/i);
+});