# Ponytail vs baseline, newest OpenAI: gpt-5.5 (top) + gpt-4.1-mini, gpt-5.4-mini.
#   npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt-newest.yaml --env-file .env --repeat 30
description: "Ponytail vs baseline, newest OpenAI (gpt-5.5 + minis). LOC + correctness, cost telemetry."

providers:
  - id: openai:gpt-5.5
    config: { max_completion_tokens: 8192 }
  - id: openai:gpt-4.1-mini
    config: { max_tokens: 8192, temperature: 1 }
  - id: openai:gpt-5.4-mini
    config: { max_completion_tokens: 8192 }

prompts:
  - id: file://arms/baseline.js
    label: baseline (no skill)
  - id: file://arms/ponytail.js
    label: ponytail

defaultTest:
  assert:
    - type: javascript
      value: file://loc.js
      metric: code_loc
    - type: javascript
      value: file://correctness.js
      metric: correct

tests:
  - vars: { task: "Write me a Python function that validates email addresses." }
  - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
  - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
  - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
  - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }