From 6d990f8c54cbca62a29cb2a7e05d1b10e8715102 Mon Sep 17 00:00:00 2001 From: at384 Date: Sun, 14 Jun 2026 23:42:01 +0200 Subject: [PATCH] feat(benchmarks): add correctness assertion (#31) * feat(benchmarks): add correctness assertion - proves less code is not broken code The existing benchmark measures lines-of-code (loc.js) but never checks whether the generated code actually works. This adds a functional correctness gate (correctness.js) that extracts code from fenced blocks and runs per-task checks: - email validator: spawns Python, asserts accept/reject on 5 inputs - debounce: spawns Node, asserts delayed execution + reset on re-call - csv sum: spawns Python with a test CSV, asserts correct total (351) - countdown (React): structural check (useState + useEffect + decrement) - rate limiter (FastAPI): structural check (limit logic + framework usage) 12 unit tests (node:test) cover good/bad outputs for every task plus the unknown-task edge case. Existing tests and rule-copy checks unaffected. * fix: address review feedback - csv check: use regex lookaround instead of substring match to prevent false positives (e.g. 13510 containing '351') - ratelimit: fix operator precedence in block finder by adding parens around the || inside the !b.lang guard - README: note that React/FastAPI checks are structural only, add prerequisites section (Python 3, pandas, Node.js 18+) - test: add regression test for csv substring false positive --- benchmarks/README.md | 15 ++ benchmarks/correctness.js | 263 ++++++++++++++++++++++++++++++++ benchmarks/promptfooconfig.yaml | 3 + tests/correctness.test.js | 191 +++++++++++++++++++++++ 4 files changed, 472 insertions(+) create mode 100644 benchmarks/correctness.js create mode 100644 tests/correctness.test.js diff --git a/benchmarks/README.md b/benchmarks/README.md index 9775fce..80e7698 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -40,6 +40,21 @@ Tasks: email validator, JS debounce, CSV sum, React countdown, FastAPI rate-limi Versus baseline, ponytail writes **80-94% less code**, costs **47-77% less**, and runs **3-6x faster**, on every model. +## Metrics + +| File | Metric | Behavior | +|------|--------|----------| +| `loc.js` | `loc` | Measurement - always passes, records line count | +| `correctness.js` | `correct` | Gate - fails if generated code doesn't work | + +`correctness.js` extracts fenced code blocks and runs per-task checks (spawns Python/Node for email, debounce, CSV; structural regex for React and FastAPI). A broken one-liner that scores great on LOC will fail on correctness. + +> **Note:** The React countdown and FastAPI rate-limit checks are keyword/structural only (no runtime execution), so they verify plausible structure rather than full correctness. The email, debounce, and CSV checks execute the code. + +### Prerequisites + +Running the benchmark requires **Python 3**, **pandas**, and **Node.js** (18+). + ## Notes - Caveman is a prose-compression skill (it leaves code "normal"), so it lands between baseline and ponytail on code size and wins mainly on prose tokens. diff --git a/benchmarks/correctness.js b/benchmarks/correctness.js new file mode 100644 index 0000000..23faccd --- /dev/null +++ b/benchmarks/correctness.js @@ -0,0 +1,263 @@ +// Functional correctness assertion: runs generated code against lightweight test +// cases per task. Proves "less code" is not "broken code". Spawns python/node +// with the extracted code + appended assertions; returns pass/fail + score. +// +// Metric: `correct` (1 = all checks pass, 0 = at least one fails). +// Unlike loc.js (measurement-only), this one is a gate — a wrong answer is a +// wrong answer regardless of how few lines produced it. + +const { execSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +// Extract fenced code blocks, tagged by language. +function extractBlocks(text) { + const matches = [...text.matchAll(/```(\w*)\n([\s\S]*?)```/g)]; + return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] })); +} + +// Identify which task we're evaluating from vars.task. +function identifyTask(task) { + const t = task.toLowerCase(); + if (t.includes('email') && t.includes('valid')) return 'email'; + if (t.includes('debounce')) return 'debounce'; + if (t.includes('csv') && t.includes('sum')) return 'csv'; + if (t.includes('countdown') && t.includes('react')) return 'countdown'; + if (t.includes('rate limit') || t.includes('rate-limit')) return 'ratelimit'; + return null; +} + +// Run a command, return { ok, stderr }. +function exec(cmd, opts = {}) { + try { + execSync(cmd, { timeout: 10_000, encoding: 'utf8', stdio: 'pipe', ...opts }); + return { ok: true, stderr: '' }; + } catch (e) { + return { ok: false, stderr: (e.stderr || e.message || '').slice(0, 500) }; + } +} + +// Write content to a temp file, return the path. +function tmpFile(ext, content) { + const p = path.join(os.tmpdir(), `ponytail-bench-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`); + fs.writeFileSync(p, content); + return p; +} + +// --- Per-task test harnesses --- + +const CHECKS = { + email(blocks) { + const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('def '))); + if (!code) return { pass: false, reason: 'No Python code block found' }; + + // Append assertions that call the generated function by common names. + const harness = ` +${code.code} + +# Find the validator function +import sys +fn = None +for name in ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate']: + if name in dir() and callable(eval(name)): + fn = eval(name) + break + +if fn is None: + # Try any function that takes one arg + import inspect + for name, obj in list(globals().items()): + if callable(obj) and not name.startswith('_'): + try: + sig = inspect.signature(obj) + if len(sig.parameters) == 1: + fn = obj + break + except (ValueError, TypeError): + pass + +if fn is None: + print("FAIL: no validator function found") + sys.exit(1) + +# Test cases +failures = [] +if not fn("user@example.com"): + failures.append("rejected valid: user@example.com") +if not fn("a@b.co"): + failures.append("rejected valid: a@b.co") +if fn("no-at-sign"): + failures.append("accepted invalid: no-at-sign") +if fn(""): + failures.append("accepted invalid: empty string") +if fn("@missing-local.com"): + failures.append("accepted invalid: @missing-local.com") + +if failures: + print("FAIL: " + "; ".join(failures)) + sys.exit(1) +print("PASS") +`; + const f = tmpFile('.py', harness); + const result = exec(`python "${f}"`); + fs.unlinkSync(f); + if (result.ok) return { pass: true, reason: 'Email validator passes all checks' }; + return { pass: false, reason: result.stderr || 'Email validator failed' }; + }, + + debounce(blocks) { + const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && b.code.includes('function'))); + if (!code) return { pass: false, reason: 'No JavaScript code block found' }; + + const harness = ` +${code.code} + +// Find the debounce function +const fn = typeof debounce === 'function' ? debounce + : typeof module !== 'undefined' && typeof module.exports === 'function' ? module.exports + : null; + +if (!fn) { + console.error("FAIL: no debounce function found"); + process.exit(1); +} + +// Test: debounced function should not fire immediately +let callCount = 0; +const debounced = fn(() => { callCount++; }, 50); +debounced(); +debounced(); +debounced(); + +if (callCount > 0) { + console.error("FAIL: debounce fired immediately (should wait)"); + process.exit(1); +} + +// Test: should fire after the delay +setTimeout(() => { + if (callCount !== 1) { + console.error("FAIL: expected 1 call after delay, got " + callCount); + process.exit(1); + } + console.log("PASS"); +}, 120); +`; + const f = tmpFile('.mjs', harness); + const result = exec(`node "${f}"`); + fs.unlinkSync(f); + if (result.ok) return { pass: true, reason: 'Debounce passes all checks' }; + return { pass: false, reason: result.stderr || 'Debounce failed' }; + }, + + csv(blocks) { + const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('csv') && b.code.includes('sum'))); + if (!code) return { pass: false, reason: 'No Python code block found' }; + + // Create a test CSV and wrap the generated code so it reads it. + const csvContent = 'name,amount\nAlice,100.5\nBob,200.0\nCharlie,50.5\n'; + const csvPath = tmpFile('.csv', csvContent).replace(/\\/g, '/'); + + // The generated code likely reads 'sales.csv'; patch the filename. + let patched = code.code.replace(/['"]sales\.csv['"]/g, `'${csvPath}'`); + // Also try open() calls + patched = patched.replace(/open\(\s*['"]sales\.csv['"]/g, `open('${csvPath}'`); + + const harness = ` +import sys, os +os.chdir(r"${path.dirname(csvPath)}") + +# Capture print output +import io +_stdout = sys.stdout +sys.stdout = io.StringIO() + +try: +${patched.split('\n').map((l) => ' ' + l).join('\n')} +except Exception as e: + sys.stdout = _stdout + # If it needs sales.csv in cwd, write it there and retry + pass + +output = sys.stdout.getvalue() +sys.stdout = _stdout + +# Check output contains the number 351 (100.5 + 200.0 + 50.5) +# Match as a standalone number (not as substring of e.g. 13510) +import re +if re.search(r'(? b.code.includes('ount') || b.code.includes('timer') || b.code.includes('Timer')); + if (!code) return { pass: false, reason: 'No countdown component found' }; + + const src = code.code; + const hasState = /useState|useReducer|this\.state/.test(src); + const hasEffect = /useEffect|componentDidMount|setInterval|setTimeout/.test(src); + const hasDecrement = /- 1|-= 1|prev - 1|count - 1|seconds - 1|time - 1/.test(src); + + const failures = []; + if (!hasState) failures.push('no state management (useState/useReducer)'); + if (!hasEffect) failures.push('no timer setup (useEffect/setInterval/setTimeout)'); + if (!hasDecrement) failures.push('no countdown decrement logic'); + + if (failures.length === 0) return { pass: true, reason: 'Countdown has required structure' }; + return { pass: false, reason: 'Missing: ' + failures.join(', ') }; + }, + + ratelimit(blocks) { + const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && (b.code.includes('rate') || b.code.includes('limit')))); + if (!code) return { pass: false, reason: 'No Python code block found' }; + + // Structural check for rate limiting: must have some form of counter/time tracking. + const src = code.code; + const hasTimeTracking = /time\.|datetime|asyncio/.test(src); + const hasLimitLogic = /limit|max_requests|rate|429|Too Many|HTTPException|RateLimiter/.test(src); + const hasFastAPI = /fastapi|FastAPI|app\s*=|@app\./.test(src); + + const failures = []; + if (!hasLimitLogic) failures.push('no rate limit logic'); + if (!hasFastAPI) failures.push('no FastAPI usage'); + + if (failures.length === 0) return { pass: true, reason: 'Rate limiter has required structure' }; + return { pass: false, reason: 'Missing: ' + failures.join(', ') }; + }, +}; + +// --- Main assertion entry point --- + +module.exports = (output, context) => { + const task = identifyTask(context.vars.task || ''); + if (!task) { + return { pass: true, score: 1, reason: 'Unknown task, skipped correctness check' }; + } + + const blocks = extractBlocks(String(output || '')); + if (blocks.length === 0) { + return { pass: false, score: 0, reason: 'No code blocks in output' }; + } + + const check = CHECKS[task]; + const result = check(blocks); + return { + pass: result.pass, + score: result.pass ? 1 : 0, + reason: result.reason, + }; +}; diff --git a/benchmarks/promptfooconfig.yaml b/benchmarks/promptfooconfig.yaml index ac4bfcb..3cb63ed 100644 --- a/benchmarks/promptfooconfig.yaml +++ b/benchmarks/promptfooconfig.yaml @@ -29,6 +29,9 @@ defaultTest: - type: javascript value: file://loc.js metric: code_loc + - type: javascript + value: file://correctness.js + metric: correct tests: - vars: { task: "Write me a Python function that validates email addresses." } diff --git a/tests/correctness.test.js b/tests/correctness.test.js new file mode 100644 index 0000000..a3facc8 --- /dev/null +++ b/tests/correctness.test.js @@ -0,0 +1,191 @@ +#!/usr/bin/env node +// Unit test for the correctness benchmark assertion. Feeds known-good and +// known-bad LLM outputs through each task checker and asserts the expected +// pass/fail verdict. Runs without promptfoo — just node:test + the module. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const correctness = require('../benchmarks/correctness'); + +// Helper: wrap code in a fenced block and call the assertion with task vars. +function check(task, lang, code) { + const output = '```' + lang + '\n' + code + '\n```'; + return correctness(output, { vars: { task } }); +} + +// --- Email validator --- + +test('email: correct one-liner passes', () => { + const result = check( + 'Write me a Python function that validates email addresses.', + 'python', + 'def validate_email(email):\n return "@" in email and "." in email.split("@")[-1] and email.split("@")[0] != ""', + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('email: always-true validator fails', () => { + const result = check( + 'Write me a Python function that validates email addresses.', + 'python', + 'def validate_email(email):\n return True', + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +test('email: no code block fails', () => { + const result = correctness('Here is my answer: just use regex.', { + vars: { task: 'Write me a Python function that validates email addresses.' }, + }); + assert.equal(result.pass, false); +}); + +// --- Debounce --- + +test('debounce: correct implementation passes', () => { + const result = check( + 'Add debounce to a search input in vanilla JavaScript.', + 'javascript', + `function debounce(fn, delay) { + let timer; + return function(...args) { + clearTimeout(timer); + timer = setTimeout(() => fn.apply(this, args), delay); + }; +}`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('debounce: immediate-call implementation fails', () => { + const result = check( + 'Add debounce to a search input in vanilla JavaScript.', + 'javascript', + `function debounce(fn, delay) { + return function(...args) { fn.apply(this, args); }; +}`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- CSV sum --- + +test('csv: correct pandas one-liner passes', () => { + const result = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `import pandas as pd +df = pd.read_csv('sales.csv') +print(df['amount'].sum())`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('csv: code that prints wrong value fails', () => { + const result = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `print(999)`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +test('csv: value containing 351 as substring fails (e.g. 13510)', () => { + const result = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `print(13510)`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- React countdown --- + +test('countdown: valid React component passes', () => { + const result = check( + 'Build me a countdown timer component in React.', + 'javascript', + `import { useState, useEffect } from 'react'; +export default function Countdown({ seconds }) { + const [count, setCount] = useState(seconds); + useEffect(() => { + if (count <= 0) return; + const id = setInterval(() => setCount(prev => prev - 1), 1000); + return () => clearInterval(id); + }, [count]); + return
{count}
; +}`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('countdown: static div without state fails', () => { + const result = check( + 'Build me a countdown timer component in React.', + 'javascript', + `export default function Countdown() { return
10
; }`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- Rate limiter --- + +test('ratelimit: FastAPI with limit logic passes', () => { + const result = check( + 'Add rate limiting to my FastAPI endpoint so users can\'t spam it.', + 'python', + `from fastapi import FastAPI, HTTPException +import time + +app = FastAPI() +requests = {} + +@app.get("/api") +def endpoint(user: str = "anon"): + now = time.time() + window = requests.get(user, []) + window = [t for t in window if now - t < 60] + if len(window) >= 10: + raise HTTPException(429, "Too Many Requests") + window.append(now) + requests[user] = window + return {"ok": True}`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('ratelimit: plain endpoint without limiting fails', () => { + const result = check( + 'Add rate limiting to my FastAPI endpoint.', + 'python', + `from fastapi import FastAPI +app = FastAPI() + +@app.get("/api") +def endpoint(): + return {"ok": True}`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- Edge cases --- + +test('unknown task is gracefully skipped', () => { + const result = correctness('```python\nprint("hi")\n```', { + vars: { task: 'Explain quantum computing.' }, + }); + assert.equal(result.pass, true); + assert.equal(result.score, 1); + assert.match(result.reason, /unknown task/i); +});