diff --git a/benchmarks/README.md b/benchmarks/README.md index a53c9df..426d4e6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -11,10 +11,13 @@ check with `node --version` and upgrade if needed): ```bash cp ../.env.example ../.env # add your ANTHROPIC_API_KEY -npx promptfoo@latest eval -c promptfooconfig.yaml --repeat 10 +npx promptfoo@latest eval -c promptfooconfig.yaml --env-file ../.env --repeat 10 npx promptfoo@latest view ``` +`--env-file ../.env` is required because promptfoo reads `.env` from the current +directory (`benchmarks/`), not the repo root where the file lives. + ### Local models via Ollama No API key or promptfoo required. Runs against any model served by Ollama: diff --git a/benchmarks/claude-email.js b/benchmarks/claude-email.js new file mode 100644 index 0000000..4a73512 --- /dev/null +++ b/benchmarks/claude-email.js @@ -0,0 +1,40 @@ +// Email under ponytail on Claude (ponytail's primary target), baseline vs ponytail. +const fs = require('fs'), path = require('path'); +const { checkPy, pyBlock, TASKS } = require('./robustness-audit.js'); +const skill = fs.readFileSync(path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8'); +const email = TASKS.find(t => t.name === 'email'); +const N = Number(process.env.CE_N) || 40; +const MODELS = (process.env.CE_MODELS || 'claude-haiku-4-5-20251001,claude-sonnet-4-6,claude-opus-4-8').split(','); + +const kv = Object.fromEntries(fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8') + .split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; })); +const KEY = kv.ANTHROPIC_API_KEY; + +async function call(model, system, user) { + const body = { model, max_tokens: 1024, messages: [{ role: 'user', content: user }] }; + if (system) body.system = system; + const r = await fetch('https://api.anthropic.com/v1/messages', { method: 'POST', + headers: { 'x-api-key': KEY, 'anthropic-version': '2023-06-01', 'content-type': 'application/json' }, body: JSON.stringify(body) }); + if (!r.ok) return { err: r.status }; + const j = await r.json(); + return { text: (j.content || []).map(b => b.text || '').join('') }; +} + +(async () => { + console.log(`email, n=${N}\n`); + console.log('model baseline ponytail'); + for (const model of MODELS) { + const rates = {}; + for (const [arm, sys] of [['baseline', null], ['ponytail', skill]]) { + let pass = 0, err = 0; + for (let i = 0; i < N; i++) { + const r = await call(model, sys, email.prompt); + if (r.err) { err++; continue; } + if (checkPy(pyBlock(r.text), email)) pass++; + } + rates[arm] = `${pass}/${N - err}`; + } + console.log(`${model.padEnd(26)} ${rates.baseline.padEnd(10)} ${rates.ponytail}`); + } +})(); diff --git a/benchmarks/correctness.js b/benchmarks/correctness.js index 5ca1753..fc56611 100644 --- a/benchmarks/correctness.js +++ b/benchmarks/correctness.js @@ -13,7 +13,11 @@ const path = require('path'); // Extract fenced code blocks, tagged by language. function extractBlocks(text) { - const matches = [...text.matchAll(/```(\w*)\n([\s\S]*?)```/g)]; + text = String(text || ''); + const matches = [...text.matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)]; + // ponytail: terse models often answer with bare, unfenced code. Treat the whole + // response as one block so the gate scores the code instead of reporting "no block". + if (matches.length === 0 && text.trim()) return [{ lang: '', code: text }]; return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] })); } @@ -121,7 +125,7 @@ print("PASS") }, debounce(blocks) { - const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && b.code.includes('function'))); + const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && (b.code.includes('function') || b.code.includes('=>')))); if (!code) return { pass: false, reason: 'No JavaScript code block found' }; const harness = ` diff --git a/benchmarks/correctness.test.js b/benchmarks/correctness.test.js new file mode 100644 index 0000000..e3103c4 --- /dev/null +++ b/benchmarks/correctness.test.js @@ -0,0 +1,26 @@ +// Regression guard for the gate fixes (issue #65). Run: node correctness.test.js +// Needs python + node on PATH, same as correctness.js itself. +const assert = require('assert'); +const check = require('./correctness.js'); + +const emailTask = { vars: { task: 'Write me a Python function that validates email addresses.' } }; +const debounceTask = { vars: { task: 'Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay).' } }; + +const FENCED_EMAIL = '```python\nimport re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))\n```'; +const UNFENCED_EMAIL = 'import re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))'; +const WRONG_EMAIL = '```python\ndef validate_email(e):\n return True # accepts everything\n```'; +const UNFENCED_ARROW_DEBOUNCE = 'const debounce = (fn, delay) => {\n let t;\n return (...a) => { clearTimeout(t); t = setTimeout(() => fn(...a), delay); };\n};'; + +let pass = 0; +const cases = [ + ['fenced email still passes', check(FENCED_EMAIL, emailTask).pass, true], + ['unfenced email now passes (bug #1 fix)', check(UNFENCED_EMAIL, emailTask).pass, true], + ['broken email still fails', check(WRONG_EMAIL, emailTask).pass, false], + ['unfenced arrow debounce passes (bug #1 + arrow-fn fix)', check(UNFENCED_ARROW_DEBOUNCE, debounceTask).pass, true], +]; +for (const [name, got, want] of cases) { + assert.strictEqual(got, want, `FAILED: ${name} (got ${got}, want ${want})`); + console.log(`ok - ${name}`); + pass++; +} +console.log(`\n${pass}/${cases.length} passed`); diff --git a/benchmarks/model-email.js b/benchmarks/model-email.js new file mode 100644 index 0000000..ea8ee8c --- /dev/null +++ b/benchmarks/model-email.js @@ -0,0 +1,39 @@ +// Cross-model email rate at high n: is the parseaddr quirk gpt-5.4-mini-specific? +const fs = require('fs'), path = require('path'); +const { checkPy, pyBlock, TASKS } = require('./robustness-audit.js'); +const skill = fs.readFileSync(path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8'); +const email = TASKS.find(t => t.name === 'email'); +const N = Number(process.env.ME_N) || 100; +const MODELS = (process.env.ME_MODELS || 'gpt-4.1-mini,gpt-5.4-mini').split(','); + +const kv = Object.fromEntries(fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8') + .split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; })); +const KEY = kv.OPENAI_API_KEY; + +async function call(model, system, user) { + const body = { model, max_completion_tokens: 4096, + messages: system ? [{ role: 'system', content: system }, { role: 'user', content: user }] : [{ role: 'user', content: user }] }; + const r = await fetch('https://api.openai.com/v1/chat/completions', { method: 'POST', + headers: { Authorization: 'Bearer ' + KEY, 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + if (!r.ok) return { err: r.status }; + return { text: (await r.json()).choices?.[0]?.message?.content || '' }; +} + +(async () => { + console.log(`email, n=${N}\n`); + console.log('model baseline ponytail'); + for (const model of MODELS) { + const rates = {}; + for (const [arm, sys] of [['baseline', null], ['ponytail', skill]]) { + let pass = 0, err = 0; + for (let i = 0; i < N; i++) { + const r = await call(model, sys, email.prompt); + if (r.err) { err++; continue; } + if (checkPy(pyBlock(r.text), email)) pass++; + } + rates[arm] = `${pass}/${N - err}`; + } + console.log(`${model.padEnd(15)} ${rates.baseline.padEnd(10)} ${rates.ponytail}`); + } +})(); diff --git a/benchmarks/promptfooconfig.gpt.yaml b/benchmarks/promptfooconfig.gpt.yaml new file mode 100644 index 0000000..f939723 --- /dev/null +++ b/benchmarks/promptfooconfig.gpt.yaml @@ -0,0 +1,32 @@ +# Reproduces Pyseph's issue-65 setup: baseline vs ponytail, gpt-4.1-mini + gpt-5.4-mini. +# Reuses the repo's arms + loc/correctness gates. Needs OPENAI_API_KEY in ../.env. +# npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt.yaml --repeat N +description: "Ponytail vs baseline on GPT-mini models (issue #65 repro). LOC + correctness gate." + +providers: + - id: openai:gpt-4.1-mini + config: { max_tokens: 4096, temperature: 1 } + - id: openai:gpt-5.4-mini + config: { max_completion_tokens: 4096 } + +prompts: + - id: file://arms/baseline.js + label: baseline (no skill) + - id: file://arms/ponytail.js + label: ponytail + +defaultTest: + assert: + - type: javascript + value: file://loc.js + metric: code_loc + - type: javascript + value: file://correctness.js + metric: correct + +tests: + - vars: { task: "Write me a Python function that validates email addresses." } + - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." } + - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." } + - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." } + - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." } diff --git a/benchmarks/promptfooconfig.yaml b/benchmarks/promptfooconfig.yaml index 3cb63ed..b624c3f 100644 --- a/benchmarks/promptfooconfig.yaml +++ b/benchmarks/promptfooconfig.yaml @@ -35,7 +35,7 @@ defaultTest: tests: - vars: { task: "Write me a Python function that validates email addresses." } - - vars: { task: "Add debounce to a search input in vanilla JavaScript. It currently fires an API call on every keystroke." } + - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." } - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." } - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." } - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." } diff --git a/benchmarks/results/2026-06-16-correctness-gate-fix.md b/benchmarks/results/2026-06-16-correctness-gate-fix.md new file mode 100644 index 0000000..7fa12db --- /dev/null +++ b/benchmarks/results/2026-06-16-correctness-gate-fix.md @@ -0,0 +1,107 @@ +# Correctness under Ponytail: gate fixes + GPT-mini reproduction (2026-06-16) + +Context: [issue #65](https://github.com/DietrichGebert/ponytail/issues/65) asked whether +Ponytail degrades model performance. A community run (Pyseph) reported a large correctness +drop on `gpt-4.1-mini` (10/15 with Ponytail vs 15/15 without) and a small one on +`gpt-5.4-mini` (14/15 vs 15/15). + +Investigating that, the correctness gate itself turned out to be the main culprit. This +writeup documents the gate bugs, the fixes, and a clean reproduction of Pyseph's exact +model setup. + +## TL;DR + +- The `correct` gate had two bugs that **under-reported correctness for terse models** — it + could not read unfenced code, and the debounce task tested for a deliverable the prompt + never asked for. +- After fixing the gate, on a clean `n=20` run of Pyseph's exact models, the large drop + **does not reproduce**: `gpt-4.1-mini` is 100% with *and* without Ponytail. +- Ponytail roughly **halves** median code size, the original headline claim, with no + meaningful correctness cost on instruction-following models. +- One genuine, small Ponytail defect surfaced and is reported honestly below. + +## The gate bugs + +1. **Unfenced code was scored as "no code blocks."** `extractBlocks()` only matched + ```` ```fenced``` ```` blocks. Models that reply with bare code (more common under + Ponytail's terse style, and frequent on `gpt-5.4-mini`) scored an automatic fail even + when the code was correct. This alone accounted for 41 of 74 failures in the first GPT run. +2. **The debounce task tested the wrong deliverable.** The prompt said *"add debounce to a + search input"* but the check expected a reusable `debounce(fn, delay)` utility it could + call. A correct inline answer (`input.addEventListener(... clearTimeout ...)`) failed with + `searchInput is not defined`. This accounted for 31 of 74 failures, and it penalized the + literal, minimal answer while rewarding code that over-built a utility nobody asked for. + +Both are fixed: `extractBlocks()` now falls back to treating the whole response as one code +block when no fence is present (and tolerates CRLF), and the debounce task now asks for the +reusable `debounce(fn, delay)` function the check actually verifies. + +## Method + +Two arms (baseline = no skill, ponytail), Pyseph's two models, the five repo tasks, `n=20` +per cell, run serially (`--max-concurrency 1`) so transient quota 429s never reduced the +denominators. Code is executed where possible (email, debounce, CSV); React/FastAPI are +structural checks (see the README caveat). Claude numbers are a free re-score of the +committed `output-10x.json` responses through the fixed gate (`n=10`, 4 tasks — the saved +debounce responses predate the prompt fix and are excluded). + +## Results + +### GPT-mini (clean `n=20`, 0 errors, full denominators) + +| model | baseline | ponytail | median LOC (base → pony) | +|---|--:|--:|--:| +| gpt-4.1-mini | 100/100 | 100/100 | 15 → 7 | +| gpt-5.4-mini | 100/100 | 98/100 | 16 → 7 | + +Pyseph's reported `gpt-4.1-mini` drop (10/15 ≈ 67%) does not reproduce — it scores 100% here. +The difference is the gate fixes; the original numbers were measuring unfenced code and the +debounce deliverable mismatch, not model degradation. + +### Claude (fixed gate, re-score of committed responses, `n=10`, 4 tasks) + +| model | baseline | ponytail | +|---|--:|--:| +| claude-haiku-4-5 | 38/40 (95%) | 40/40 (100%) | +| claude-opus-4-8 | 40/40 (100%) | 40/40 (100%) | +| claude-sonnet-4-6 | 28/40 (70%) | 40/40 (100%) | + +On instruction-following models Ponytail ties or slightly *beats* baseline. The low +`sonnet` baseline number is itself an over-engineering failure: the unconstrained validator +returns a rich `{is_valid, message}` dict instead of a bool, so `if validate_email(addr)` is +always truthy and accepts every address — a real bug Ponytail's `return bool(...)` avoids. + +## The one real Ponytail defect + +On `gpt-5.4-mini`, 2 of 20 Ponytail email runs failed because the model reached for the +laziest stdlib option: + +```python +from email.utils import parseaddr +def is_valid_email(email): + _, addr = parseaddr(email) + return addr == email and "@" in addr # accepts "@missing-local.com" +``` + +`parseaddr` does not require a local part, so `"@missing-local.com"` is accepted. This is a +genuine (if minor) cost of pushing toward one-liners: occasionally the chosen stdlib helper +has an edge-case hole. The other 18 runs used a regex and passed. + +## Reproduce + +```bash +# GPT arms (needs OPENAI_API_KEY in ../.env) +cd benchmarks +npx promptfoo@latest eval -c promptfooconfig.gpt.yaml --env-file ../.env --repeat 20 --max-concurrency 1 + +# Claude re-score of committed responses through the fixed gate +node -e 'const c=require("./correctness.js"),d=require("./output-10x.json");/* score d.results.results through c */' +``` + +## Takeaway + +The "Ponytail hurts correctness" reports trace to a benchmark that could not read terse +output, not to the skill. With the gate fixed, the LOC win holds and correctness is flat on +capable models. The honest caveats remain: the effect is model-dependent (small/local models +follow the ladder poorly — see the llama3.2 writeup), and chasing the shortest answer can +occasionally pick a stdlib helper with an edge-case gap. diff --git a/benchmarks/results/2026-06-16-robustness-audit.md b/benchmarks/results/2026-06-16-robustness-audit.md new file mode 100644 index 0000000..c30bf73 --- /dev/null +++ b/benchmarks/results/2026-06-16-robustness-audit.md @@ -0,0 +1,129 @@ +# Robustness audit: does ponytail degrade weak models? (2026-06-16) + +Follow-up to [issue #65](https://github.com/DietrichGebert/ponytail/issues/65). After fixing +the correctness-gate bugs, the open question was the real one: does Ponytail's push toward +the shortest solution make weak models produce *wrong* code on edge cases? This audit +answers it directly, with a deliberately hostile test set and high sample counts. + +## TL;DR + +- Across **12 classic edge-case traps** (off-by-one, n=0, leap-century, subtractive Roman, + deep nesting, …) on **two weak models** (`gpt-4.1-mini`, `gpt-5.4-mini`), Ponytail holds + **baseline parity** — it does not produce more wrong answers than the unconstrained model. +- The **one** measured soft spot is email validation, and it is **provider-specific**. + OpenAI models, at every size, sometimes reach for `email.utils.parseaddr` (a parser, not a + validator) under "stdlib-first" pressure and accept `"@missing-local.com"`. On Claude, + ponytail's target platform, email is **100%** (haiku/sonnet/opus, n=40 each). +- The slip is **not fixable by skill text**: 8 distinct SKILL.md edits (including an n=100 + A/B, 96% → 95%) all scored ≤ the current skill, several worse, all bloating LOC. Counter- + instructions make small models overthink and fail *more*. Nothing was shipped — adding + skill text that doesn't move the number is exactly the cargo-cult Ponytail exists to avoid. + +## Method + +`baseline` (no skill) vs `ponytail` (full SKILL.md), single-shot, default params, +`gpt-4.1-mini` and `gpt-5.4-mini`. Each task runs generated code against edge-case +assertions. Every check is **self-verified**: a known-correct and a known-lazy-wrong +reference must pass/fail respectively before any model output is scored +(`node robustness-audit.js --selftest`, 16/16). Runs were serial to avoid quota 429s +shrinking denominators. + +## Edge-case traps (n=20/cell) + +All 12 algorithmic tasks: `baseline 20/20 == ponytail 20/20` on **both** models. Examples +of the traps (the lazy version passes the common case, fails the edge): + +| task | the trap a lazy impl misses | +|---|---| +| is_prime | n = 0, 1, negatives | +| factorial / fibonacci | n = 0 | +| binary_search | empty list, target at the last index (off-by-one) | +| is_leap_year / days_in_month | 1900 not leap, 2000 leap (century rule) | +| int_to_roman | subtractive forms (4=IV, 9=IX, 40=XL) | +| flatten | nesting deeper than one level | +| clamp | value already in range | +| chunk | trailing remainder | + +The only sub-20 cell in the first run was `gpt-5.4-mini` flatten at 19/20 — a single +stochastic miss that **did not reproduce**: 50/50 at n=50. (`clamp` showed 19/19, i.e. one +API error, not a wrong answer.) + +## Validators: the email slip is provider-specific + +The one place ponytail measurably affects correctness is **email validation**, via the +parse ≠ validate trap: under "stdlib-first" pressure a model reaches for +`email.utils.parseaddr` — a *parser* that accepts malformed input like `@missing-local.com` +— instead of writing an explicit check. The split is by **provider**, not model size. + +**OpenAI (email, baseline vs ponytail, n=50–100):** + +| model | baseline | ponytail | +|---|--:|--:| +| gpt-4.1-mini | 100% | 98% | +| gpt-4.1 | 100% | 79% | +| gpt-5.4-mini | ~100% | ~92% | +| gpt-5.4 | 100% | 98% | +| gpt-5.5 | 98% | 94% | + +**Claude (email, baseline vs ponytail, n=40):** + +| model | baseline | ponytail | +|---|--:|--:| +| claude-haiku-4-5 | 35/40 | **40/40** | +| claude-sonnet-4-6 | 0/40 * | **40/40** | +| claude-opus-4-8 | 39/40 | **40/40** | + +Every OpenAI model slips regardless of size (gpt-4.1 full is the worst). Every Claude model +is **100%** under ponytail. + +\* The Sonnet baseline `0/40` is a return-type artifact, not a logic failure, and should not +be read as "Sonnet cannot validate email." Unconstrained Sonnet over-engineers the validator +into a `dict` (`{is_valid, message}`) instead of a bool. The test calls the function as a +bool, and a non-empty dict is always truthy, so it "accepts" every address and scores 0. +Read dict-aware (via `is_valid`), its logic is about 75% correct (9/12). The honest point is +narrow: ponytail writes the plain correct bool the task implies, while the unconstrained +model over-builds the interface and trips a naive `if validate(x)` caller. `url`, +`creditcard`, and `ipv4` hold at ~100% under ponytail on both providers, because their lazy +stdlib choice (`ipaddress`, Luhn, scheme checks) is already strict. Only email's obvious +stdlib tool is a parser. + +## The fix that wasn't + +SKILL.md already says "never simplify away input validation" and "pick the stdlib option +correct on edge cases." We tried hard to push the OpenAI rate to 100% by editing the skill — +**8 distinct edits** across counter-pressure wording, a check-mandate, explicit-over-delegate, +a few-shot example, combinations, and three placements. Every one scored ≤ the current skill; +several were far worse (one cratered to 78%); all bloated median LOC. The definitive n=100 +A/B of the most promising edit: + +``` +OLD skill: 96/100 (96.0%) +NEW skill: 95/100 (95.0%) -> within noise, no reliable effect +``` + +Counter-instructions backfire: piling validation rules onto the skill makes models overthink +and produce *more* broken validators, not fewer. The reflex to reach for `parseaddr` lives in +the OpenAI models' training, and no skill wording reliably overrides it — so nothing was +shipped. Adding skill text that doesn't work is the cargo-cult Ponytail exists to prevent. + +## Conclusion + +"Ponytail degrades model performance" is not supported. Across 12 edge-case traps, ponytail +holds baseline parity. On validation it is **100% on every Claude model**, which is its +target platform. The only blemish is an email-validator slip on OpenAI models (a +cross-provider `parseaddr` reflex, present at every size), documented here and not fixable by +skill text. The LOC win (about half the code) comes with no correctness tax on Claude. + +## Reproduce + +```bash +cd benchmarks +node robustness-audit.js --selftest # verify all 16 instruments (no API) +node robustness-audit.js # 16-task audit, gpt-5.4-mini, n=20 +AUDIT_MODEL=gpt-4.1-mini node robustness-audit.js + +# email cross-provider (the slip) +ME_MODELS="gpt-4.1,gpt-5.4,gpt-5.5" ME_N=50 node model-email.js # OpenAI (OPENAI_API_KEY) +node claude-email.js # Claude (ANTHROPIC_API_KEY) +``` +`OPENAI_API_KEY` / `ANTHROPIC_API_KEY` read from `../.env`. diff --git a/benchmarks/robustness-audit.js b/benchmarks/robustness-audit.js new file mode 100644 index 0000000..1b05e08 --- /dev/null +++ b/benchmarks/robustness-audit.js @@ -0,0 +1,194 @@ +// Robustness audit (issue #65 follow-up): find where ponytail actually breaks on a +// weak model. 12 tasks with classic edge-case traps. Each has a known-good and a +// known-lazy-wrong reference so the instrument is verified before any API spend. +// node robustness-audit.js --selftest # no API: prove every check is correct +// node robustness-audit.js # baseline vs ponytail, gpt-5.4-mini, n=20 +const { execSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const N = Number(process.env.AUDIT_N) || 20; +const MODEL = process.env.AUDIT_MODEL || 'gpt-5.4-mini'; +const ROOT = path.join(__dirname, '..'); +let kv = {}; +try { + kv = Object.fromEntries(fs.readFileSync(path.join(ROOT, '.env'), 'utf8') + .split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; })); +} catch (_) { /* no .env — fine for --selftest */ } +const KEY = process.env.OPENAI_API_KEY || kv.OPENAI_API_KEY; +const SKILL = fs.readFileSync(path.join(ROOT, 'skills', 'ponytail', 'SKILL.md'), 'utf8'); + +// task = { name, prompt, names, arity, cases: [[argsArray, expected], ...], good, bad } +const TASKS = [ + { name: 'is_prime', arity: 1, names: ['is_prime', 'isprime', 'prime'], + prompt: 'Write a Python function is_prime(n) that returns True if n is prime, else False.', + cases: [[[2], true], [[1], false], [[0], false], [[-7], false], [[17], true], [[15], false], [[97], true]], + good: 'def is_prime(n):\n if n < 2: return False\n for i in range(2, int(n**0.5)+1):\n if n % i == 0: return False\n return True', + bad: 'def is_prime(n):\n for i in range(2, n):\n if n % i == 0: return False\n return True' }, + { name: 'factorial', arity: 1, names: ['factorial', 'fact'], + prompt: 'Write a Python function factorial(n).', + cases: [[[0], 1], [[1], 1], [[5], 120], [[6], 720]], + good: 'def factorial(n):\n r = 1\n for i in range(2, n+1): r *= i\n return r', + bad: 'def factorial(n):\n r = 1\n for i in range(1, n): r *= i\n return r' }, + { name: 'fibonacci', arity: 1, names: ['fibonacci', 'fib'], + prompt: 'Write fibonacci(n) returning the nth Fibonacci number, with fib(0)=0 and fib(1)=1.', + cases: [[[0], 0], [[1], 1], [[2], 1], [[7], 13], [[10], 55]], + good: 'def fibonacci(n):\n a, b = 0, 1\n for _ in range(n): a, b = b, a+b\n return a', + bad: 'def fibonacci(n):\n a, b = 1, 1\n for _ in range(n): a, b = b, a+b\n return a' }, + { name: 'gcd', arity: 2, names: ['gcd'], + prompt: 'Write gcd(a, b) returning the greatest common divisor.', + cases: [[[12, 8], 4], [[5, 0], 5], [[0, 5], 5], [[17, 5], 1], [[100, 75], 25]], + good: 'def gcd(a, b):\n while b: a, b = b, a % b\n return a', + bad: 'def gcd(a, b):\n for i in range(min(a, b), 0, -1):\n if a % i == 0 and b % i == 0: return i' }, + { name: 'binary_search', arity: 2, names: ['binary_search', 'bsearch', 'search'], + prompt: 'Write binary_search(arr, target) returning the index of target in the sorted list arr, or -1 if absent.', + cases: [[[[1, 2, 3, 4, 5], 3], 2], [[[1, 2, 3, 4, 5], 1], 0], [[[1, 2, 3, 4, 5], 5], 4], [[[1, 2, 3, 4, 5], 6], -1], [[[], 1], -1], [[[1], 1], 0]], + good: 'def binary_search(arr, target):\n lo, hi = 0, len(arr)-1\n while lo <= hi:\n m = (lo+hi)//2\n if arr[m] == target: return m\n elif arr[m] < target: lo = m+1\n else: hi = m-1\n return -1', + bad: 'def binary_search(arr, target):\n lo, hi = 0, len(arr)-1\n while lo < hi:\n m = (lo+hi)//2\n if arr[m] == target: return m\n elif arr[m] < target: lo = m+1\n else: hi = m-1\n return -1' }, + { name: 'is_leap_year', arity: 1, names: ['is_leap_year', 'is_leap', 'leap'], + prompt: 'Write is_leap_year(year) returning True if it is a leap year.', + cases: [[[2000], true], [[1900], false], [[2020], true], [[2021], false], [[2400], true], [[2100], false]], + good: 'def is_leap_year(y):\n return y % 4 == 0 and (y % 100 != 0 or y % 400 == 0)', + bad: 'def is_leap_year(y):\n return y % 4 == 0' }, + { name: 'days_in_month', arity: 2, names: ['days_in_month'], + prompt: 'Write days_in_month(year, month) returning the number of days in that month.', + cases: [[[2020, 2], 29], [[2021, 2], 28], [[1900, 2], 28], [[2000, 2], 29], [[2021, 4], 30], [[2021, 1], 31], [[2021, 12], 31]], + good: 'import calendar\ndef days_in_month(year, month):\n return calendar.monthrange(year, month)[1]', + bad: 'def days_in_month(year, month):\n return [31,28,31,30,31,30,31,31,30,31,30,31][month-1]' }, + { name: 'int_to_roman', arity: 1, names: ['int_to_roman', 'to_roman', 'roman'], + prompt: 'Write int_to_roman(n) converting an integer (1-3999) to its Roman numeral string.', + cases: [[[4], 'IV'], [[9], 'IX'], [[58], 'LVIII'], [[1994], 'MCMXCIV'], [[40], 'XL'], [[3], 'III']], + good: "def int_to_roman(n):\n vals=[(1000,'M'),(900,'CM'),(500,'D'),(400,'CD'),(100,'C'),(90,'XC'),(50,'L'),(40,'XL'),(10,'X'),(9,'IX'),(5,'V'),(4,'IV'),(1,'I')]\n r=''\n for v,s in vals:\n while n>=v: r+=s; n-=v\n return r", + bad: "def int_to_roman(n):\n vals=[(1000,'M'),(500,'D'),(100,'C'),(50,'L'),(10,'X'),(5,'V'),(1,'I')]\n r=''\n for v,s in vals:\n while n>=v: r+=s; n-=v\n return r" }, + { name: 'flatten', arity: 1, names: ['flatten'], + prompt: 'Write flatten(lst) that fully flattens an arbitrarily nested list of integers into a flat list.', + cases: [[[[1, [2, [3, 4]], 5]], [1, 2, 3, 4, 5]], [[[]], []], [[[1, 2, 3]], [1, 2, 3]], [[[1, [2], [[3]]]], [1, 2, 3]]], + good: 'def flatten(lst):\n out = []\n for x in lst:\n if isinstance(x, list): out.extend(flatten(x))\n else: out.append(x)\n return out', + bad: 'def flatten(lst):\n return [x for s in lst for x in (s if isinstance(s, list) else [s])]' }, + { name: 'chunk', arity: 2, names: ['chunk'], + prompt: 'Write chunk(lst, size) splitting lst into consecutive sublists of length size (the last may be shorter).', + cases: [[[[1, 2, 3, 4, 5], 2], [[1, 2], [3, 4], [5]]], [[[1, 2, 3, 4], 2], [[1, 2], [3, 4]]], [[[], 3], []], [[[1], 5], [[1]]]], + good: 'def chunk(lst, size):\n return [lst[i:i+size] for i in range(0, len(lst), size)]', + bad: 'def chunk(lst, size):\n return [lst[i:i+size] for i in range(0, len(lst)-size+1, size)]' }, + { name: 'clamp', arity: 3, names: ['clamp'], + prompt: 'Write clamp(value, low, high) returning value bounded to the range [low, high].', + cases: [[[5, 0, 10], 5], [[-1, 0, 10], 0], [[15, 0, 10], 10], [[0, 0, 10], 0], [[10, 0, 10], 10]], + good: 'def clamp(value, low, high):\n return max(low, min(value, high))', + bad: 'def clamp(value, low, high):\n if value < low: return low\n if value > high: return high' }, + { name: 'is_palindrome', arity: 1, names: ['is_palindrome', 'palindrome', 'is_pal'], + prompt: 'Write is_palindrome(s) returning True if s is a palindrome, ignoring case, spaces, and punctuation.', + cases: [[['racecar'], true], [['A man, a plan, a canal: Panama'], true], [['hello'], false], [[''], true], [["No 'x' in Nixon"], true], [['ab'], false]], + good: "def is_palindrome(s):\n c = [ch.lower() for ch in s if ch.isalnum()]\n return c == c[::-1]", + bad: 'def is_palindrome(s):\n return s == s[::-1]' }, + // Validators: the parse != validate trap. email is ponytail's one measured soft spot + // on gpt-5.4-mini (~4-5%, parseaddr); the rest hold parity. See results writeup. + { name: 'email', arity: 1, names: ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate'], + prompt: 'Write me a Python function that validates email addresses.', + cases: [[['user@example.com'], true], [['a@b.co'], true], [['no-at-sign'], false], [[''], false], [['@missing-local.com'], false]], + good: 'import re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))', + bad: 'from email.utils import parseaddr\ndef validate_email(e):\n _, a = parseaddr(e)\n return a == e and "@" in a' }, + { name: 'url', arity: 1, names: ['validate_url', 'is_valid_url', 'is_url', 'validate', 'is_valid'], + prompt: 'Write a Python function that validates whether a string is a valid HTTP or HTTPS URL.', + cases: [[['https://example.com'], true], [['http://a.b/c'], true], [['https://x.io/p?q=1'], true], [['garbage'], false], [[''], false], [['example.com'], false], [['ftp://example.com'], false], [['http://'], false]], + good: 'from urllib.parse import urlparse\ndef validate_url(u):\n p = urlparse(u)\n return p.scheme in ("http","https") and bool(p.netloc)', + bad: 'from urllib.parse import urlparse\ndef validate_url(u):\n return bool(urlparse(u))' }, + { name: 'creditcard', arity: 1, names: ['validate_credit_card', 'is_valid_card', 'validate_card', 'luhn', 'validate', 'is_valid'], + prompt: 'Write a Python function that validates a credit card number.', + cases: [[['4242424242424242'], true], [['4012888888881881'], true], [['4242424242424241'], false], [['12345'], false], [['abcd'], false]], + good: 'def validate_credit_card(n):\n d=[int(c) for c in str(n) if c.isdigit()]\n if len(d)<13: return False\n s=0\n for i,x in enumerate(reversed(d)):\n if i%2==1:\n x*=2\n if x>9: x-=9\n s+=x\n return s%10==0', + bad: "def validate_credit_card(n):\n s=str(n).replace(' ','')\n return s.isdigit() and len(s)==16" }, + { name: 'ipv4', arity: 1, names: ['validate_ipv4', 'is_valid_ip', 'is_ipv4', 'validate_ip', 'validate', 'is_valid'], + prompt: 'Write a Python function that validates an IPv4 address.', + cases: [[['192.168.1.1'], true], [['0.0.0.0'], true], [['255.255.255.255'], true], [['999.999.999.999'], false], [['256.1.1.1'], false], [['1.2.3'], false], [['abc'], false]], + good: 'import ipaddress\ndef validate_ipv4(s):\n try:\n ipaddress.IPv4Address(s); return True\n except Exception: return False', + bad: "import re\ndef validate_ipv4(s):\n return bool(re.match(r'^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$', s))" }, +]; + +function pyBlock(text) { + const m = [...String(text || '').matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)]; + if (!m.length) return text || ''; + const py = m.find(x => /py/.test(x[1])); + return (py || m[0])[2]; +} + +function checkPy(code, task) { + const harness = `import sys, json, inspect +${code} +TARGET = ${task.arity} +names = json.loads(r'''${JSON.stringify(task.names)}''') +fn = None +for nm in names: + if nm in dir() and callable(eval(nm)): fn = eval(nm); break +if fn is None: + for nm, obj in list(globals().items()): + if callable(obj) and not nm.startswith('_') and not inspect.isclass(obj): + try: + if len(inspect.signature(obj).parameters) == TARGET: fn = obj; break + except (ValueError, TypeError): pass +if fn is None: print('NOFN'); sys.exit(1) +cases = json.loads(r'''${JSON.stringify(task.cases)}''') +for args, expected in cases: + try: r = fn(*args) + except Exception as e: print('EXC', args, e); sys.exit(1) + if r != expected: print('MISMATCH', args, '->', r, 'want', expected); sys.exit(1) +print('PASS')`; + const f = path.join(os.tmpdir(), `audit-${process.pid}-${Math.random().toString(36).slice(2)}.py`); + fs.writeFileSync(f, harness); + try { execSync(`python3 "${f}"`, { timeout: 10000, encoding: 'utf8', stdio: 'pipe' }); return true; } + catch (e) { return false; } + finally { try { fs.unlinkSync(f); } catch (_) {} } +} + +async function call(system, user) { + const body = { model: MODEL, max_completion_tokens: 4096, + messages: system ? [{ role: 'system', content: system }, { role: 'user', content: user }] : [{ role: 'user', content: user }] }; + const r = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', headers: { Authorization: 'Bearer ' + KEY, 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + if (!r.ok) return { err: r.status }; + const j = await r.json(); + return { text: j.choices?.[0]?.message?.content || '' }; +} + +module.exports = { checkPy, pyBlock, call, TASKS, SKILL }; +if (require.main !== module) return; + +if (process.argv.includes('--selftest')) { + let ok = 0, bad = 0; + for (const t of TASKS) { + const g = checkPy(t.good, t), b = checkPy(t.bad, t); + const pass = g === true && b === false; + console.log(`${pass ? 'ok ' : 'XX '} ${t.name.padEnd(16)} good=${g} bad=${b}`); + pass ? ok++ : bad++; + } + console.log(`\nself-test: ${ok}/${TASKS.length} instruments valid${bad ? ` — ${bad} BROKEN` : ''}`); + process.exit(bad ? 1 : 0); +} + +(async () => { + const arms = { baseline: null, ponytail: SKILL }; + const grid = {}; + for (const t of TASKS) { + grid[t.name] = {}; + for (const arm of Object.keys(arms)) { + let pass = 0, err = 0; + for (let i = 0; i < N; i++) { + const res = await call(arms[arm], t.prompt); + if (res.err) { err++; continue; } + if (checkPy(pyBlock(res.text), t)) pass++; + } + grid[t.name][arm] = { pass, n: N - err }; + } + const b = grid[t.name].baseline, p = grid[t.name].ponytail; + const flag = p.pass < b.pass ? ' <-- PONYTAIL REGRESSION' : (p.pass < p.n ? ' (both imperfect)' : ''); + console.log(`${t.name.padEnd(16)} baseline ${b.pass}/${b.n} ponytail ${p.pass}/${p.n}${flag}`); + } + console.log('\n=== ponytail holes (ponytail < baseline) ==='); + let any = false; + for (const t of TASKS) { + const b = grid[t.name].baseline, p = grid[t.name].ponytail; + if (p.pass < b.pass) { console.log(` ${t.name}: ${b.pass} -> ${p.pass}`); any = true; } + } + if (!any) console.log(' none'); +})();