* fix(benchmarks): correctness gate scores unfenced code; fix debounce task The `correct` gate under-reported correctness for terse models, the likely source of "Ponytail degrades models" reports (issue #65): - extractBlocks() only matched fenced code blocks, so bare/unfenced code scored an automatic fail even when correct. Now falls back to the whole response as one block (and tolerates CRLF). Debounce detection also accepts unfenced arrow functions. - The debounce task asked to "add debounce to a search input" but the check expected a reusable debounce(fn, delay) util, failing correct inline answers. Task reworded to the deliverable the check verifies. Adds correctness.test.js (regression guard) and a GPT-mini repro config plus results writeup: on a clean n=20 run, the reported gpt-4.1-mini drop (10/15) does not reproduce (100/100). The LOC win (~halved) holds. README repro fixed: promptfoo needs --env-file ../.env (reads cwd, not root). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test(benchmarks): add robustness audit — ponytail vs baseline on edge cases Answers the real question behind #65: does ponytail's push for the shortest solution make weak models produce wrong code on edge cases? robustness-audit.js: 16 self-verifying tasks (12 algorithmic edge-case traps + 4 validators). Each check ships a known-good and known-lazy-wrong reference that must pass/fail before any model output is scored (--selftest, 16/16). Findings (gpt-4.1-mini + gpt-5.4-mini, baseline vs ponytail): parity on every edge-case trap on both models. The one measured soft spot is gpt-5.4-mini email (~4-5%, reaches for parseaddr). A sharpened SKILL.md validation rule had no reliable effect in an n=100 A/B (96% vs 95%), so it was not shipped — the tendency is model-level, not skill-level. Full writeup in results/. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test(benchmarks): email slip is provider-specific — 100% on Claude High-n cross-provider follow-up to the robustness audit. The one ponytail soft spot (email validation via parseaddr) splits by provider, not model size: - Claude (haiku/sonnet/opus): 100% under ponytail, n=40 each — and ponytail beats baseline (unconstrained Sonnet over-engineers into an always-truthy dict, 0/40; ponytail writes a clean validator). - OpenAI (gpt-4.1-mini..gpt-5.5): slips at every size under ponytail (~79-98%), baseline ~100%. The parseaddr reflex lives in OpenAI training. Not fixable by skill text: 8 distinct SKILL.md edits (incl. an n=100 A/B, 96% vs 95%) all scored <= current, several worse, all bloated LOC. Nothing shipped. SKILL.md unchanged. Conclusion: on ponytail's target platform (Claude) email is 100%; the GPT slip is a documented cross-provider transfer quirk. Adds model-email.js / claude-email.js to reproduce the tables. Writeup updated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs(benchmarks): correct misleading Sonnet baseline 0 percent The Sonnet baseline 0/40 on email is a return-type artifact, not a logic failure: unconstrained Sonnet returns a dict {is_valid, message} instead of a bool, so the bool-contract gate scores every case as accepted. Read dict-aware via is_valid, its logic is ~75% correct (9/12). Reframed honestly so we are not presenting 0 vs 100 as a clean win; ponytail still wins (clean 100% bool) but the point is over-engineered return type, not total failure. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
DietrichGebert
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
687c1b3398
commit
caf138df56
@@ -11,10 +11,13 @@ check with `node --version` and upgrade if needed):
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp ../.env.example ../.env # add your ANTHROPIC_API_KEY
|
cp ../.env.example ../.env # add your ANTHROPIC_API_KEY
|
||||||
npx promptfoo@latest eval -c promptfooconfig.yaml --repeat 10
|
npx promptfoo@latest eval -c promptfooconfig.yaml --env-file ../.env --repeat 10
|
||||||
npx promptfoo@latest view
|
npx promptfoo@latest view
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`--env-file ../.env` is required because promptfoo reads `.env` from the current
|
||||||
|
directory (`benchmarks/`), not the repo root where the file lives.
|
||||||
|
|
||||||
### Local models via Ollama
|
### Local models via Ollama
|
||||||
|
|
||||||
No API key or promptfoo required. Runs against any model served by Ollama:
|
No API key or promptfoo required. Runs against any model served by Ollama:
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
// Email under ponytail on Claude (ponytail's primary target), baseline vs ponytail.
|
||||||
|
const fs = require('fs'), path = require('path');
|
||||||
|
const { checkPy, pyBlock, TASKS } = require('./robustness-audit.js');
|
||||||
|
const skill = fs.readFileSync(path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8');
|
||||||
|
const email = TASKS.find(t => t.name === 'email');
|
||||||
|
const N = Number(process.env.CE_N) || 40;
|
||||||
|
const MODELS = (process.env.CE_MODELS || 'claude-haiku-4-5-20251001,claude-sonnet-4-6,claude-opus-4-8').split(',');
|
||||||
|
|
||||||
|
const kv = Object.fromEntries(fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8')
|
||||||
|
.split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#'))
|
||||||
|
.map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; }));
|
||||||
|
const KEY = kv.ANTHROPIC_API_KEY;
|
||||||
|
|
||||||
|
async function call(model, system, user) {
|
||||||
|
const body = { model, max_tokens: 1024, messages: [{ role: 'user', content: user }] };
|
||||||
|
if (system) body.system = system;
|
||||||
|
const r = await fetch('https://api.anthropic.com/v1/messages', { method: 'POST',
|
||||||
|
headers: { 'x-api-key': KEY, 'anthropic-version': '2023-06-01', 'content-type': 'application/json' }, body: JSON.stringify(body) });
|
||||||
|
if (!r.ok) return { err: r.status };
|
||||||
|
const j = await r.json();
|
||||||
|
return { text: (j.content || []).map(b => b.text || '').join('') };
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
console.log(`email, n=${N}\n`);
|
||||||
|
console.log('model baseline ponytail');
|
||||||
|
for (const model of MODELS) {
|
||||||
|
const rates = {};
|
||||||
|
for (const [arm, sys] of [['baseline', null], ['ponytail', skill]]) {
|
||||||
|
let pass = 0, err = 0;
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const r = await call(model, sys, email.prompt);
|
||||||
|
if (r.err) { err++; continue; }
|
||||||
|
if (checkPy(pyBlock(r.text), email)) pass++;
|
||||||
|
}
|
||||||
|
rates[arm] = `${pass}/${N - err}`;
|
||||||
|
}
|
||||||
|
console.log(`${model.padEnd(26)} ${rates.baseline.padEnd(10)} ${rates.ponytail}`);
|
||||||
|
}
|
||||||
|
})();
|
||||||
@@ -13,7 +13,11 @@ const path = require('path');
|
|||||||
|
|
||||||
// Extract fenced code blocks, tagged by language.
|
// Extract fenced code blocks, tagged by language.
|
||||||
function extractBlocks(text) {
|
function extractBlocks(text) {
|
||||||
const matches = [...text.matchAll(/```(\w*)\n([\s\S]*?)```/g)];
|
text = String(text || '');
|
||||||
|
const matches = [...text.matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)];
|
||||||
|
// ponytail: terse models often answer with bare, unfenced code. Treat the whole
|
||||||
|
// response as one block so the gate scores the code instead of reporting "no block".
|
||||||
|
if (matches.length === 0 && text.trim()) return [{ lang: '', code: text }];
|
||||||
return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] }));
|
return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] }));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,7 +125,7 @@ print("PASS")
|
|||||||
},
|
},
|
||||||
|
|
||||||
debounce(blocks) {
|
debounce(blocks) {
|
||||||
const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && b.code.includes('function')));
|
const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && (b.code.includes('function') || b.code.includes('=>'))));
|
||||||
if (!code) return { pass: false, reason: 'No JavaScript code block found' };
|
if (!code) return { pass: false, reason: 'No JavaScript code block found' };
|
||||||
|
|
||||||
const harness = `
|
const harness = `
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
// Regression guard for the gate fixes (issue #65). Run: node correctness.test.js
|
||||||
|
// Needs python + node on PATH, same as correctness.js itself.
|
||||||
|
const assert = require('assert');
|
||||||
|
const check = require('./correctness.js');
|
||||||
|
|
||||||
|
const emailTask = { vars: { task: 'Write me a Python function that validates email addresses.' } };
|
||||||
|
const debounceTask = { vars: { task: 'Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay).' } };
|
||||||
|
|
||||||
|
const FENCED_EMAIL = '```python\nimport re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))\n```';
|
||||||
|
const UNFENCED_EMAIL = 'import re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))';
|
||||||
|
const WRONG_EMAIL = '```python\ndef validate_email(e):\n return True # accepts everything\n```';
|
||||||
|
const UNFENCED_ARROW_DEBOUNCE = 'const debounce = (fn, delay) => {\n let t;\n return (...a) => { clearTimeout(t); t = setTimeout(() => fn(...a), delay); };\n};';
|
||||||
|
|
||||||
|
let pass = 0;
|
||||||
|
const cases = [
|
||||||
|
['fenced email still passes', check(FENCED_EMAIL, emailTask).pass, true],
|
||||||
|
['unfenced email now passes (bug #1 fix)', check(UNFENCED_EMAIL, emailTask).pass, true],
|
||||||
|
['broken email still fails', check(WRONG_EMAIL, emailTask).pass, false],
|
||||||
|
['unfenced arrow debounce passes (bug #1 + arrow-fn fix)', check(UNFENCED_ARROW_DEBOUNCE, debounceTask).pass, true],
|
||||||
|
];
|
||||||
|
for (const [name, got, want] of cases) {
|
||||||
|
assert.strictEqual(got, want, `FAILED: ${name} (got ${got}, want ${want})`);
|
||||||
|
console.log(`ok - ${name}`);
|
||||||
|
pass++;
|
||||||
|
}
|
||||||
|
console.log(`\n${pass}/${cases.length} passed`);
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
// Cross-model email rate at high n: is the parseaddr quirk gpt-5.4-mini-specific?
|
||||||
|
const fs = require('fs'), path = require('path');
|
||||||
|
const { checkPy, pyBlock, TASKS } = require('./robustness-audit.js');
|
||||||
|
const skill = fs.readFileSync(path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8');
|
||||||
|
const email = TASKS.find(t => t.name === 'email');
|
||||||
|
const N = Number(process.env.ME_N) || 100;
|
||||||
|
const MODELS = (process.env.ME_MODELS || 'gpt-4.1-mini,gpt-5.4-mini').split(',');
|
||||||
|
|
||||||
|
const kv = Object.fromEntries(fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8')
|
||||||
|
.split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#'))
|
||||||
|
.map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; }));
|
||||||
|
const KEY = kv.OPENAI_API_KEY;
|
||||||
|
|
||||||
|
async function call(model, system, user) {
|
||||||
|
const body = { model, max_completion_tokens: 4096,
|
||||||
|
messages: system ? [{ role: 'system', content: system }, { role: 'user', content: user }] : [{ role: 'user', content: user }] };
|
||||||
|
const r = await fetch('https://api.openai.com/v1/chat/completions', { method: 'POST',
|
||||||
|
headers: { Authorization: 'Bearer ' + KEY, 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||||
|
if (!r.ok) return { err: r.status };
|
||||||
|
return { text: (await r.json()).choices?.[0]?.message?.content || '' };
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
console.log(`email, n=${N}\n`);
|
||||||
|
console.log('model baseline ponytail');
|
||||||
|
for (const model of MODELS) {
|
||||||
|
const rates = {};
|
||||||
|
for (const [arm, sys] of [['baseline', null], ['ponytail', skill]]) {
|
||||||
|
let pass = 0, err = 0;
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const r = await call(model, sys, email.prompt);
|
||||||
|
if (r.err) { err++; continue; }
|
||||||
|
if (checkPy(pyBlock(r.text), email)) pass++;
|
||||||
|
}
|
||||||
|
rates[arm] = `${pass}/${N - err}`;
|
||||||
|
}
|
||||||
|
console.log(`${model.padEnd(15)} ${rates.baseline.padEnd(10)} ${rates.ponytail}`);
|
||||||
|
}
|
||||||
|
})();
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# Reproduces Pyseph's issue-65 setup: baseline vs ponytail, gpt-4.1-mini + gpt-5.4-mini.
|
||||||
|
# Reuses the repo's arms + loc/correctness gates. Needs OPENAI_API_KEY in ../.env.
|
||||||
|
# npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt.yaml --repeat N
|
||||||
|
description: "Ponytail vs baseline on GPT-mini models (issue #65 repro). LOC + correctness gate."
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- id: openai:gpt-4.1-mini
|
||||||
|
config: { max_tokens: 4096, temperature: 1 }
|
||||||
|
- id: openai:gpt-5.4-mini
|
||||||
|
config: { max_completion_tokens: 4096 }
|
||||||
|
|
||||||
|
prompts:
|
||||||
|
- id: file://arms/baseline.js
|
||||||
|
label: baseline (no skill)
|
||||||
|
- id: file://arms/ponytail.js
|
||||||
|
label: ponytail
|
||||||
|
|
||||||
|
defaultTest:
|
||||||
|
assert:
|
||||||
|
- type: javascript
|
||||||
|
value: file://loc.js
|
||||||
|
metric: code_loc
|
||||||
|
- type: javascript
|
||||||
|
value: file://correctness.js
|
||||||
|
metric: correct
|
||||||
|
|
||||||
|
tests:
|
||||||
|
- vars: { task: "Write me a Python function that validates email addresses." }
|
||||||
|
- vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
|
||||||
|
- vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
|
||||||
|
- vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
|
||||||
|
- vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }
|
||||||
@@ -35,7 +35,7 @@ defaultTest:
|
|||||||
|
|
||||||
tests:
|
tests:
|
||||||
- vars: { task: "Write me a Python function that validates email addresses." }
|
- vars: { task: "Write me a Python function that validates email addresses." }
|
||||||
- vars: { task: "Add debounce to a search input in vanilla JavaScript. It currently fires an API call on every keystroke." }
|
- vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
|
||||||
- vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
|
- vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
|
||||||
- vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
|
- vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
|
||||||
- vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }
|
- vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
# Correctness under Ponytail: gate fixes + GPT-mini reproduction (2026-06-16)
|
||||||
|
|
||||||
|
Context: [issue #65](https://github.com/DietrichGebert/ponytail/issues/65) asked whether
|
||||||
|
Ponytail degrades model performance. A community run (Pyseph) reported a large correctness
|
||||||
|
drop on `gpt-4.1-mini` (10/15 with Ponytail vs 15/15 without) and a small one on
|
||||||
|
`gpt-5.4-mini` (14/15 vs 15/15).
|
||||||
|
|
||||||
|
Investigating that, the correctness gate itself turned out to be the main culprit. This
|
||||||
|
writeup documents the gate bugs, the fixes, and a clean reproduction of Pyseph's exact
|
||||||
|
model setup.
|
||||||
|
|
||||||
|
## TL;DR
|
||||||
|
|
||||||
|
- The `correct` gate had two bugs that **under-reported correctness for terse models** — it
|
||||||
|
could not read unfenced code, and the debounce task tested for a deliverable the prompt
|
||||||
|
never asked for.
|
||||||
|
- After fixing the gate, on a clean `n=20` run of Pyseph's exact models, the large drop
|
||||||
|
**does not reproduce**: `gpt-4.1-mini` is 100% with *and* without Ponytail.
|
||||||
|
- Ponytail roughly **halves** median code size, the original headline claim, with no
|
||||||
|
meaningful correctness cost on instruction-following models.
|
||||||
|
- One genuine, small Ponytail defect surfaced and is reported honestly below.
|
||||||
|
|
||||||
|
## The gate bugs
|
||||||
|
|
||||||
|
1. **Unfenced code was scored as "no code blocks."** `extractBlocks()` only matched
|
||||||
|
```` ```fenced``` ```` blocks. Models that reply with bare code (more common under
|
||||||
|
Ponytail's terse style, and frequent on `gpt-5.4-mini`) scored an automatic fail even
|
||||||
|
when the code was correct. This alone accounted for 41 of 74 failures in the first GPT run.
|
||||||
|
2. **The debounce task tested the wrong deliverable.** The prompt said *"add debounce to a
|
||||||
|
search input"* but the check expected a reusable `debounce(fn, delay)` utility it could
|
||||||
|
call. A correct inline answer (`input.addEventListener(... clearTimeout ...)`) failed with
|
||||||
|
`searchInput is not defined`. This accounted for 31 of 74 failures, and it penalized the
|
||||||
|
literal, minimal answer while rewarding code that over-built a utility nobody asked for.
|
||||||
|
|
||||||
|
Both are fixed: `extractBlocks()` now falls back to treating the whole response as one code
|
||||||
|
block when no fence is present (and tolerates CRLF), and the debounce task now asks for the
|
||||||
|
reusable `debounce(fn, delay)` function the check actually verifies.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
Two arms (baseline = no skill, ponytail), Pyseph's two models, the five repo tasks, `n=20`
|
||||||
|
per cell, run serially (`--max-concurrency 1`) so transient quota 429s never reduced the
|
||||||
|
denominators. Code is executed where possible (email, debounce, CSV); React/FastAPI are
|
||||||
|
structural checks (see the README caveat). Claude numbers are a free re-score of the
|
||||||
|
committed `output-10x.json` responses through the fixed gate (`n=10`, 4 tasks — the saved
|
||||||
|
debounce responses predate the prompt fix and are excluded).
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
### GPT-mini (clean `n=20`, 0 errors, full denominators)
|
||||||
|
|
||||||
|
| model | baseline | ponytail | median LOC (base → pony) |
|
||||||
|
|---|--:|--:|--:|
|
||||||
|
| gpt-4.1-mini | 100/100 | 100/100 | 15 → 7 |
|
||||||
|
| gpt-5.4-mini | 100/100 | 98/100 | 16 → 7 |
|
||||||
|
|
||||||
|
Pyseph's reported `gpt-4.1-mini` drop (10/15 ≈ 67%) does not reproduce — it scores 100% here.
|
||||||
|
The difference is the gate fixes; the original numbers were measuring unfenced code and the
|
||||||
|
debounce deliverable mismatch, not model degradation.
|
||||||
|
|
||||||
|
### Claude (fixed gate, re-score of committed responses, `n=10`, 4 tasks)
|
||||||
|
|
||||||
|
| model | baseline | ponytail |
|
||||||
|
|---|--:|--:|
|
||||||
|
| claude-haiku-4-5 | 38/40 (95%) | 40/40 (100%) |
|
||||||
|
| claude-opus-4-8 | 40/40 (100%) | 40/40 (100%) |
|
||||||
|
| claude-sonnet-4-6 | 28/40 (70%) | 40/40 (100%) |
|
||||||
|
|
||||||
|
On instruction-following models Ponytail ties or slightly *beats* baseline. The low
|
||||||
|
`sonnet` baseline number is itself an over-engineering failure: the unconstrained validator
|
||||||
|
returns a rich `{is_valid, message}` dict instead of a bool, so `if validate_email(addr)` is
|
||||||
|
always truthy and accepts every address — a real bug Ponytail's `return bool(...)` avoids.
|
||||||
|
|
||||||
|
## The one real Ponytail defect
|
||||||
|
|
||||||
|
On `gpt-5.4-mini`, 2 of 20 Ponytail email runs failed because the model reached for the
|
||||||
|
laziest stdlib option:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from email.utils import parseaddr
|
||||||
|
def is_valid_email(email):
|
||||||
|
_, addr = parseaddr(email)
|
||||||
|
return addr == email and "@" in addr # accepts "@missing-local.com"
|
||||||
|
```
|
||||||
|
|
||||||
|
`parseaddr` does not require a local part, so `"@missing-local.com"` is accepted. This is a
|
||||||
|
genuine (if minor) cost of pushing toward one-liners: occasionally the chosen stdlib helper
|
||||||
|
has an edge-case hole. The other 18 runs used a regex and passed.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# GPT arms (needs OPENAI_API_KEY in ../.env)
|
||||||
|
cd benchmarks
|
||||||
|
npx promptfoo@latest eval -c promptfooconfig.gpt.yaml --env-file ../.env --repeat 20 --max-concurrency 1
|
||||||
|
|
||||||
|
# Claude re-score of committed responses through the fixed gate
|
||||||
|
node -e 'const c=require("./correctness.js"),d=require("./output-10x.json");/* score d.results.results through c */'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Takeaway
|
||||||
|
|
||||||
|
The "Ponytail hurts correctness" reports trace to a benchmark that could not read terse
|
||||||
|
output, not to the skill. With the gate fixed, the LOC win holds and correctness is flat on
|
||||||
|
capable models. The honest caveats remain: the effect is model-dependent (small/local models
|
||||||
|
follow the ladder poorly — see the llama3.2 writeup), and chasing the shortest answer can
|
||||||
|
occasionally pick a stdlib helper with an edge-case gap.
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
# Robustness audit: does ponytail degrade weak models? (2026-06-16)
|
||||||
|
|
||||||
|
Follow-up to [issue #65](https://github.com/DietrichGebert/ponytail/issues/65). After fixing
|
||||||
|
the correctness-gate bugs, the open question was the real one: does Ponytail's push toward
|
||||||
|
the shortest solution make weak models produce *wrong* code on edge cases? This audit
|
||||||
|
answers it directly, with a deliberately hostile test set and high sample counts.
|
||||||
|
|
||||||
|
## TL;DR
|
||||||
|
|
||||||
|
- Across **12 classic edge-case traps** (off-by-one, n=0, leap-century, subtractive Roman,
|
||||||
|
deep nesting, …) on **two weak models** (`gpt-4.1-mini`, `gpt-5.4-mini`), Ponytail holds
|
||||||
|
**baseline parity** — it does not produce more wrong answers than the unconstrained model.
|
||||||
|
- The **one** measured soft spot is email validation, and it is **provider-specific**.
|
||||||
|
OpenAI models, at every size, sometimes reach for `email.utils.parseaddr` (a parser, not a
|
||||||
|
validator) under "stdlib-first" pressure and accept `"@missing-local.com"`. On Claude,
|
||||||
|
ponytail's target platform, email is **100%** (haiku/sonnet/opus, n=40 each).
|
||||||
|
- The slip is **not fixable by skill text**: 8 distinct SKILL.md edits (including an n=100
|
||||||
|
A/B, 96% → 95%) all scored ≤ the current skill, several worse, all bloating LOC. Counter-
|
||||||
|
instructions make small models overthink and fail *more*. Nothing was shipped — adding
|
||||||
|
skill text that doesn't move the number is exactly the cargo-cult Ponytail exists to avoid.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
`baseline` (no skill) vs `ponytail` (full SKILL.md), single-shot, default params,
|
||||||
|
`gpt-4.1-mini` and `gpt-5.4-mini`. Each task runs generated code against edge-case
|
||||||
|
assertions. Every check is **self-verified**: a known-correct and a known-lazy-wrong
|
||||||
|
reference must pass/fail respectively before any model output is scored
|
||||||
|
(`node robustness-audit.js --selftest`, 16/16). Runs were serial to avoid quota 429s
|
||||||
|
shrinking denominators.
|
||||||
|
|
||||||
|
## Edge-case traps (n=20/cell)
|
||||||
|
|
||||||
|
All 12 algorithmic tasks: `baseline 20/20 == ponytail 20/20` on **both** models. Examples
|
||||||
|
of the traps (the lazy version passes the common case, fails the edge):
|
||||||
|
|
||||||
|
| task | the trap a lazy impl misses |
|
||||||
|
|---|---|
|
||||||
|
| is_prime | n = 0, 1, negatives |
|
||||||
|
| factorial / fibonacci | n = 0 |
|
||||||
|
| binary_search | empty list, target at the last index (off-by-one) |
|
||||||
|
| is_leap_year / days_in_month | 1900 not leap, 2000 leap (century rule) |
|
||||||
|
| int_to_roman | subtractive forms (4=IV, 9=IX, 40=XL) |
|
||||||
|
| flatten | nesting deeper than one level |
|
||||||
|
| clamp | value already in range |
|
||||||
|
| chunk | trailing remainder |
|
||||||
|
|
||||||
|
The only sub-20 cell in the first run was `gpt-5.4-mini` flatten at 19/20 — a single
|
||||||
|
stochastic miss that **did not reproduce**: 50/50 at n=50. (`clamp` showed 19/19, i.e. one
|
||||||
|
API error, not a wrong answer.)
|
||||||
|
|
||||||
|
## Validators: the email slip is provider-specific
|
||||||
|
|
||||||
|
The one place ponytail measurably affects correctness is **email validation**, via the
|
||||||
|
parse ≠ validate trap: under "stdlib-first" pressure a model reaches for
|
||||||
|
`email.utils.parseaddr` — a *parser* that accepts malformed input like `@missing-local.com`
|
||||||
|
— instead of writing an explicit check. The split is by **provider**, not model size.
|
||||||
|
|
||||||
|
**OpenAI (email, baseline vs ponytail, n=50–100):**
|
||||||
|
|
||||||
|
| model | baseline | ponytail |
|
||||||
|
|---|--:|--:|
|
||||||
|
| gpt-4.1-mini | 100% | 98% |
|
||||||
|
| gpt-4.1 | 100% | 79% |
|
||||||
|
| gpt-5.4-mini | ~100% | ~92% |
|
||||||
|
| gpt-5.4 | 100% | 98% |
|
||||||
|
| gpt-5.5 | 98% | 94% |
|
||||||
|
|
||||||
|
**Claude (email, baseline vs ponytail, n=40):**
|
||||||
|
|
||||||
|
| model | baseline | ponytail |
|
||||||
|
|---|--:|--:|
|
||||||
|
| claude-haiku-4-5 | 35/40 | **40/40** |
|
||||||
|
| claude-sonnet-4-6 | 0/40 * | **40/40** |
|
||||||
|
| claude-opus-4-8 | 39/40 | **40/40** |
|
||||||
|
|
||||||
|
Every OpenAI model slips regardless of size (gpt-4.1 full is the worst). Every Claude model
|
||||||
|
is **100%** under ponytail.
|
||||||
|
|
||||||
|
\* The Sonnet baseline `0/40` is a return-type artifact, not a logic failure, and should not
|
||||||
|
be read as "Sonnet cannot validate email." Unconstrained Sonnet over-engineers the validator
|
||||||
|
into a `dict` (`{is_valid, message}`) instead of a bool. The test calls the function as a
|
||||||
|
bool, and a non-empty dict is always truthy, so it "accepts" every address and scores 0.
|
||||||
|
Read dict-aware (via `is_valid`), its logic is about 75% correct (9/12). The honest point is
|
||||||
|
narrow: ponytail writes the plain correct bool the task implies, while the unconstrained
|
||||||
|
model over-builds the interface and trips a naive `if validate(x)` caller. `url`,
|
||||||
|
`creditcard`, and `ipv4` hold at ~100% under ponytail on both providers, because their lazy
|
||||||
|
stdlib choice (`ipaddress`, Luhn, scheme checks) is already strict. Only email's obvious
|
||||||
|
stdlib tool is a parser.
|
||||||
|
|
||||||
|
## The fix that wasn't
|
||||||
|
|
||||||
|
SKILL.md already says "never simplify away input validation" and "pick the stdlib option
|
||||||
|
correct on edge cases." We tried hard to push the OpenAI rate to 100% by editing the skill —
|
||||||
|
**8 distinct edits** across counter-pressure wording, a check-mandate, explicit-over-delegate,
|
||||||
|
a few-shot example, combinations, and three placements. Every one scored ≤ the current skill;
|
||||||
|
several were far worse (one cratered to 78%); all bloated median LOC. The definitive n=100
|
||||||
|
A/B of the most promising edit:
|
||||||
|
|
||||||
|
```
|
||||||
|
OLD skill: 96/100 (96.0%)
|
||||||
|
NEW skill: 95/100 (95.0%) -> within noise, no reliable effect
|
||||||
|
```
|
||||||
|
|
||||||
|
Counter-instructions backfire: piling validation rules onto the skill makes models overthink
|
||||||
|
and produce *more* broken validators, not fewer. The reflex to reach for `parseaddr` lives in
|
||||||
|
the OpenAI models' training, and no skill wording reliably overrides it — so nothing was
|
||||||
|
shipped. Adding skill text that doesn't work is the cargo-cult Ponytail exists to prevent.
|
||||||
|
|
||||||
|
## Conclusion
|
||||||
|
|
||||||
|
"Ponytail degrades model performance" is not supported. Across 12 edge-case traps, ponytail
|
||||||
|
holds baseline parity. On validation it is **100% on every Claude model**, which is its
|
||||||
|
target platform. The only blemish is an email-validator slip on OpenAI models (a
|
||||||
|
cross-provider `parseaddr` reflex, present at every size), documented here and not fixable by
|
||||||
|
skill text. The LOC win (about half the code) comes with no correctness tax on Claude.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd benchmarks
|
||||||
|
node robustness-audit.js --selftest # verify all 16 instruments (no API)
|
||||||
|
node robustness-audit.js # 16-task audit, gpt-5.4-mini, n=20
|
||||||
|
AUDIT_MODEL=gpt-4.1-mini node robustness-audit.js
|
||||||
|
|
||||||
|
# email cross-provider (the slip)
|
||||||
|
ME_MODELS="gpt-4.1,gpt-5.4,gpt-5.5" ME_N=50 node model-email.js # OpenAI (OPENAI_API_KEY)
|
||||||
|
node claude-email.js # Claude (ANTHROPIC_API_KEY)
|
||||||
|
```
|
||||||
|
`OPENAI_API_KEY` / `ANTHROPIC_API_KEY` read from `../.env`.
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
// Robustness audit (issue #65 follow-up): find where ponytail actually breaks on a
|
||||||
|
// weak model. 12 tasks with classic edge-case traps. Each has a known-good and a
|
||||||
|
// known-lazy-wrong reference so the instrument is verified before any API spend.
|
||||||
|
// node robustness-audit.js --selftest # no API: prove every check is correct
|
||||||
|
// node robustness-audit.js # baseline vs ponytail, gpt-5.4-mini, n=20
|
||||||
|
const { execSync } = require('child_process');
|
||||||
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
|
const path = require('path');
|
||||||
|
|
||||||
|
const N = Number(process.env.AUDIT_N) || 20;
|
||||||
|
const MODEL = process.env.AUDIT_MODEL || 'gpt-5.4-mini';
|
||||||
|
const ROOT = path.join(__dirname, '..');
|
||||||
|
let kv = {};
|
||||||
|
try {
|
||||||
|
kv = Object.fromEntries(fs.readFileSync(path.join(ROOT, '.env'), 'utf8')
|
||||||
|
.split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#'))
|
||||||
|
.map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; }));
|
||||||
|
} catch (_) { /* no .env — fine for --selftest */ }
|
||||||
|
const KEY = process.env.OPENAI_API_KEY || kv.OPENAI_API_KEY;
|
||||||
|
const SKILL = fs.readFileSync(path.join(ROOT, 'skills', 'ponytail', 'SKILL.md'), 'utf8');
|
||||||
|
|
||||||
|
// task = { name, prompt, names, arity, cases: [[argsArray, expected], ...], good, bad }
|
||||||
|
const TASKS = [
|
||||||
|
{ name: 'is_prime', arity: 1, names: ['is_prime', 'isprime', 'prime'],
|
||||||
|
prompt: 'Write a Python function is_prime(n) that returns True if n is prime, else False.',
|
||||||
|
cases: [[[2], true], [[1], false], [[0], false], [[-7], false], [[17], true], [[15], false], [[97], true]],
|
||||||
|
good: 'def is_prime(n):\n if n < 2: return False\n for i in range(2, int(n**0.5)+1):\n if n % i == 0: return False\n return True',
|
||||||
|
bad: 'def is_prime(n):\n for i in range(2, n):\n if n % i == 0: return False\n return True' },
|
||||||
|
{ name: 'factorial', arity: 1, names: ['factorial', 'fact'],
|
||||||
|
prompt: 'Write a Python function factorial(n).',
|
||||||
|
cases: [[[0], 1], [[1], 1], [[5], 120], [[6], 720]],
|
||||||
|
good: 'def factorial(n):\n r = 1\n for i in range(2, n+1): r *= i\n return r',
|
||||||
|
bad: 'def factorial(n):\n r = 1\n for i in range(1, n): r *= i\n return r' },
|
||||||
|
{ name: 'fibonacci', arity: 1, names: ['fibonacci', 'fib'],
|
||||||
|
prompt: 'Write fibonacci(n) returning the nth Fibonacci number, with fib(0)=0 and fib(1)=1.',
|
||||||
|
cases: [[[0], 0], [[1], 1], [[2], 1], [[7], 13], [[10], 55]],
|
||||||
|
good: 'def fibonacci(n):\n a, b = 0, 1\n for _ in range(n): a, b = b, a+b\n return a',
|
||||||
|
bad: 'def fibonacci(n):\n a, b = 1, 1\n for _ in range(n): a, b = b, a+b\n return a' },
|
||||||
|
{ name: 'gcd', arity: 2, names: ['gcd'],
|
||||||
|
prompt: 'Write gcd(a, b) returning the greatest common divisor.',
|
||||||
|
cases: [[[12, 8], 4], [[5, 0], 5], [[0, 5], 5], [[17, 5], 1], [[100, 75], 25]],
|
||||||
|
good: 'def gcd(a, b):\n while b: a, b = b, a % b\n return a',
|
||||||
|
bad: 'def gcd(a, b):\n for i in range(min(a, b), 0, -1):\n if a % i == 0 and b % i == 0: return i' },
|
||||||
|
{ name: 'binary_search', arity: 2, names: ['binary_search', 'bsearch', 'search'],
|
||||||
|
prompt: 'Write binary_search(arr, target) returning the index of target in the sorted list arr, or -1 if absent.',
|
||||||
|
cases: [[[[1, 2, 3, 4, 5], 3], 2], [[[1, 2, 3, 4, 5], 1], 0], [[[1, 2, 3, 4, 5], 5], 4], [[[1, 2, 3, 4, 5], 6], -1], [[[], 1], -1], [[[1], 1], 0]],
|
||||||
|
good: 'def binary_search(arr, target):\n lo, hi = 0, len(arr)-1\n while lo <= hi:\n m = (lo+hi)//2\n if arr[m] == target: return m\n elif arr[m] < target: lo = m+1\n else: hi = m-1\n return -1',
|
||||||
|
bad: 'def binary_search(arr, target):\n lo, hi = 0, len(arr)-1\n while lo < hi:\n m = (lo+hi)//2\n if arr[m] == target: return m\n elif arr[m] < target: lo = m+1\n else: hi = m-1\n return -1' },
|
||||||
|
{ name: 'is_leap_year', arity: 1, names: ['is_leap_year', 'is_leap', 'leap'],
|
||||||
|
prompt: 'Write is_leap_year(year) returning True if it is a leap year.',
|
||||||
|
cases: [[[2000], true], [[1900], false], [[2020], true], [[2021], false], [[2400], true], [[2100], false]],
|
||||||
|
good: 'def is_leap_year(y):\n return y % 4 == 0 and (y % 100 != 0 or y % 400 == 0)',
|
||||||
|
bad: 'def is_leap_year(y):\n return y % 4 == 0' },
|
||||||
|
{ name: 'days_in_month', arity: 2, names: ['days_in_month'],
|
||||||
|
prompt: 'Write days_in_month(year, month) returning the number of days in that month.',
|
||||||
|
cases: [[[2020, 2], 29], [[2021, 2], 28], [[1900, 2], 28], [[2000, 2], 29], [[2021, 4], 30], [[2021, 1], 31], [[2021, 12], 31]],
|
||||||
|
good: 'import calendar\ndef days_in_month(year, month):\n return calendar.monthrange(year, month)[1]',
|
||||||
|
bad: 'def days_in_month(year, month):\n return [31,28,31,30,31,30,31,31,30,31,30,31][month-1]' },
|
||||||
|
{ name: 'int_to_roman', arity: 1, names: ['int_to_roman', 'to_roman', 'roman'],
|
||||||
|
prompt: 'Write int_to_roman(n) converting an integer (1-3999) to its Roman numeral string.',
|
||||||
|
cases: [[[4], 'IV'], [[9], 'IX'], [[58], 'LVIII'], [[1994], 'MCMXCIV'], [[40], 'XL'], [[3], 'III']],
|
||||||
|
good: "def int_to_roman(n):\n vals=[(1000,'M'),(900,'CM'),(500,'D'),(400,'CD'),(100,'C'),(90,'XC'),(50,'L'),(40,'XL'),(10,'X'),(9,'IX'),(5,'V'),(4,'IV'),(1,'I')]\n r=''\n for v,s in vals:\n while n>=v: r+=s; n-=v\n return r",
|
||||||
|
bad: "def int_to_roman(n):\n vals=[(1000,'M'),(500,'D'),(100,'C'),(50,'L'),(10,'X'),(5,'V'),(1,'I')]\n r=''\n for v,s in vals:\n while n>=v: r+=s; n-=v\n return r" },
|
||||||
|
{ name: 'flatten', arity: 1, names: ['flatten'],
|
||||||
|
prompt: 'Write flatten(lst) that fully flattens an arbitrarily nested list of integers into a flat list.',
|
||||||
|
cases: [[[[1, [2, [3, 4]], 5]], [1, 2, 3, 4, 5]], [[[]], []], [[[1, 2, 3]], [1, 2, 3]], [[[1, [2], [[3]]]], [1, 2, 3]]],
|
||||||
|
good: 'def flatten(lst):\n out = []\n for x in lst:\n if isinstance(x, list): out.extend(flatten(x))\n else: out.append(x)\n return out',
|
||||||
|
bad: 'def flatten(lst):\n return [x for s in lst for x in (s if isinstance(s, list) else [s])]' },
|
||||||
|
{ name: 'chunk', arity: 2, names: ['chunk'],
|
||||||
|
prompt: 'Write chunk(lst, size) splitting lst into consecutive sublists of length size (the last may be shorter).',
|
||||||
|
cases: [[[[1, 2, 3, 4, 5], 2], [[1, 2], [3, 4], [5]]], [[[1, 2, 3, 4], 2], [[1, 2], [3, 4]]], [[[], 3], []], [[[1], 5], [[1]]]],
|
||||||
|
good: 'def chunk(lst, size):\n return [lst[i:i+size] for i in range(0, len(lst), size)]',
|
||||||
|
bad: 'def chunk(lst, size):\n return [lst[i:i+size] for i in range(0, len(lst)-size+1, size)]' },
|
||||||
|
{ name: 'clamp', arity: 3, names: ['clamp'],
|
||||||
|
prompt: 'Write clamp(value, low, high) returning value bounded to the range [low, high].',
|
||||||
|
cases: [[[5, 0, 10], 5], [[-1, 0, 10], 0], [[15, 0, 10], 10], [[0, 0, 10], 0], [[10, 0, 10], 10]],
|
||||||
|
good: 'def clamp(value, low, high):\n return max(low, min(value, high))',
|
||||||
|
bad: 'def clamp(value, low, high):\n if value < low: return low\n if value > high: return high' },
|
||||||
|
{ name: 'is_palindrome', arity: 1, names: ['is_palindrome', 'palindrome', 'is_pal'],
|
||||||
|
prompt: 'Write is_palindrome(s) returning True if s is a palindrome, ignoring case, spaces, and punctuation.',
|
||||||
|
cases: [[['racecar'], true], [['A man, a plan, a canal: Panama'], true], [['hello'], false], [[''], true], [["No 'x' in Nixon"], true], [['ab'], false]],
|
||||||
|
good: "def is_palindrome(s):\n c = [ch.lower() for ch in s if ch.isalnum()]\n return c == c[::-1]",
|
||||||
|
bad: 'def is_palindrome(s):\n return s == s[::-1]' },
|
||||||
|
// Validators: the parse != validate trap. email is ponytail's one measured soft spot
|
||||||
|
// on gpt-5.4-mini (~4-5%, parseaddr); the rest hold parity. See results writeup.
|
||||||
|
{ name: 'email', arity: 1, names: ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate'],
|
||||||
|
prompt: 'Write me a Python function that validates email addresses.',
|
||||||
|
cases: [[['user@example.com'], true], [['a@b.co'], true], [['no-at-sign'], false], [[''], false], [['@missing-local.com'], false]],
|
||||||
|
good: 'import re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))',
|
||||||
|
bad: 'from email.utils import parseaddr\ndef validate_email(e):\n _, a = parseaddr(e)\n return a == e and "@" in a' },
|
||||||
|
{ name: 'url', arity: 1, names: ['validate_url', 'is_valid_url', 'is_url', 'validate', 'is_valid'],
|
||||||
|
prompt: 'Write a Python function that validates whether a string is a valid HTTP or HTTPS URL.',
|
||||||
|
cases: [[['https://example.com'], true], [['http://a.b/c'], true], [['https://x.io/p?q=1'], true], [['garbage'], false], [[''], false], [['example.com'], false], [['ftp://example.com'], false], [['http://'], false]],
|
||||||
|
good: 'from urllib.parse import urlparse\ndef validate_url(u):\n p = urlparse(u)\n return p.scheme in ("http","https") and bool(p.netloc)',
|
||||||
|
bad: 'from urllib.parse import urlparse\ndef validate_url(u):\n return bool(urlparse(u))' },
|
||||||
|
{ name: 'creditcard', arity: 1, names: ['validate_credit_card', 'is_valid_card', 'validate_card', 'luhn', 'validate', 'is_valid'],
|
||||||
|
prompt: 'Write a Python function that validates a credit card number.',
|
||||||
|
cases: [[['4242424242424242'], true], [['4012888888881881'], true], [['4242424242424241'], false], [['12345'], false], [['abcd'], false]],
|
||||||
|
good: 'def validate_credit_card(n):\n d=[int(c) for c in str(n) if c.isdigit()]\n if len(d)<13: return False\n s=0\n for i,x in enumerate(reversed(d)):\n if i%2==1:\n x*=2\n if x>9: x-=9\n s+=x\n return s%10==0',
|
||||||
|
bad: "def validate_credit_card(n):\n s=str(n).replace(' ','')\n return s.isdigit() and len(s)==16" },
|
||||||
|
{ name: 'ipv4', arity: 1, names: ['validate_ipv4', 'is_valid_ip', 'is_ipv4', 'validate_ip', 'validate', 'is_valid'],
|
||||||
|
prompt: 'Write a Python function that validates an IPv4 address.',
|
||||||
|
cases: [[['192.168.1.1'], true], [['0.0.0.0'], true], [['255.255.255.255'], true], [['999.999.999.999'], false], [['256.1.1.1'], false], [['1.2.3'], false], [['abc'], false]],
|
||||||
|
good: 'import ipaddress\ndef validate_ipv4(s):\n try:\n ipaddress.IPv4Address(s); return True\n except Exception: return False',
|
||||||
|
bad: "import re\ndef validate_ipv4(s):\n return bool(re.match(r'^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$', s))" },
|
||||||
|
];
|
||||||
|
|
||||||
|
function pyBlock(text) {
|
||||||
|
const m = [...String(text || '').matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)];
|
||||||
|
if (!m.length) return text || '';
|
||||||
|
const py = m.find(x => /py/.test(x[1]));
|
||||||
|
return (py || m[0])[2];
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkPy(code, task) {
|
||||||
|
const harness = `import sys, json, inspect
|
||||||
|
${code}
|
||||||
|
TARGET = ${task.arity}
|
||||||
|
names = json.loads(r'''${JSON.stringify(task.names)}''')
|
||||||
|
fn = None
|
||||||
|
for nm in names:
|
||||||
|
if nm in dir() and callable(eval(nm)): fn = eval(nm); break
|
||||||
|
if fn is None:
|
||||||
|
for nm, obj in list(globals().items()):
|
||||||
|
if callable(obj) and not nm.startswith('_') and not inspect.isclass(obj):
|
||||||
|
try:
|
||||||
|
if len(inspect.signature(obj).parameters) == TARGET: fn = obj; break
|
||||||
|
except (ValueError, TypeError): pass
|
||||||
|
if fn is None: print('NOFN'); sys.exit(1)
|
||||||
|
cases = json.loads(r'''${JSON.stringify(task.cases)}''')
|
||||||
|
for args, expected in cases:
|
||||||
|
try: r = fn(*args)
|
||||||
|
except Exception as e: print('EXC', args, e); sys.exit(1)
|
||||||
|
if r != expected: print('MISMATCH', args, '->', r, 'want', expected); sys.exit(1)
|
||||||
|
print('PASS')`;
|
||||||
|
const f = path.join(os.tmpdir(), `audit-${process.pid}-${Math.random().toString(36).slice(2)}.py`);
|
||||||
|
fs.writeFileSync(f, harness);
|
||||||
|
try { execSync(`python3 "${f}"`, { timeout: 10000, encoding: 'utf8', stdio: 'pipe' }); return true; }
|
||||||
|
catch (e) { return false; }
|
||||||
|
finally { try { fs.unlinkSync(f); } catch (_) {} }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function call(system, user) {
|
||||||
|
const body = { model: MODEL, max_completion_tokens: 4096,
|
||||||
|
messages: system ? [{ role: 'system', content: system }, { role: 'user', content: user }] : [{ role: 'user', content: user }] };
|
||||||
|
const r = await fetch('https://api.openai.com/v1/chat/completions', {
|
||||||
|
method: 'POST', headers: { Authorization: 'Bearer ' + KEY, 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||||
|
if (!r.ok) return { err: r.status };
|
||||||
|
const j = await r.json();
|
||||||
|
return { text: j.choices?.[0]?.message?.content || '' };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { checkPy, pyBlock, call, TASKS, SKILL };
|
||||||
|
if (require.main !== module) return;
|
||||||
|
|
||||||
|
if (process.argv.includes('--selftest')) {
|
||||||
|
let ok = 0, bad = 0;
|
||||||
|
for (const t of TASKS) {
|
||||||
|
const g = checkPy(t.good, t), b = checkPy(t.bad, t);
|
||||||
|
const pass = g === true && b === false;
|
||||||
|
console.log(`${pass ? 'ok ' : 'XX '} ${t.name.padEnd(16)} good=${g} bad=${b}`);
|
||||||
|
pass ? ok++ : bad++;
|
||||||
|
}
|
||||||
|
console.log(`\nself-test: ${ok}/${TASKS.length} instruments valid${bad ? ` — ${bad} BROKEN` : ''}`);
|
||||||
|
process.exit(bad ? 1 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const arms = { baseline: null, ponytail: SKILL };
|
||||||
|
const grid = {};
|
||||||
|
for (const t of TASKS) {
|
||||||
|
grid[t.name] = {};
|
||||||
|
for (const arm of Object.keys(arms)) {
|
||||||
|
let pass = 0, err = 0;
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const res = await call(arms[arm], t.prompt);
|
||||||
|
if (res.err) { err++; continue; }
|
||||||
|
if (checkPy(pyBlock(res.text), t)) pass++;
|
||||||
|
}
|
||||||
|
grid[t.name][arm] = { pass, n: N - err };
|
||||||
|
}
|
||||||
|
const b = grid[t.name].baseline, p = grid[t.name].ponytail;
|
||||||
|
const flag = p.pass < b.pass ? ' <-- PONYTAIL REGRESSION' : (p.pass < p.n ? ' (both imperfect)' : '');
|
||||||
|
console.log(`${t.name.padEnd(16)} baseline ${b.pass}/${b.n} ponytail ${p.pass}/${p.n}${flag}`);
|
||||||
|
}
|
||||||
|
console.log('\n=== ponytail holes (ponytail < baseline) ===');
|
||||||
|
let any = false;
|
||||||
|
for (const t of TASKS) {
|
||||||
|
const b = grid[t.name].baseline, p = grid[t.name].ponytail;
|
||||||
|
if (p.pass < b.pass) { console.log(` ${t.name}: ${b.pass} -> ${p.pass}`); any = true; }
|
||||||
|
}
|
||||||
|
if (!any) console.log(' none');
|
||||||
|
})();
|
||||||
Reference in New Issue
Block a user