# Ponytail benchmark: code size + cost across three arms, same model, same tasks.
#
#   Run:   npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml
#   View:  npx promptfoo@latest view
#   Share: npx promptfoo@latest share   (publishes a hosted report URL)
#
# Needs ANTHROPIC_API_KEY in the environment or a .env file (see benchmarks/README.md).
# Caveman arm uses JuliusBrussee/caveman SKILL.md (MIT), vendored at arms/caveman-SKILL.md.
description: "Ponytail vs caveman vs no-skill: same model, same tasks. Measures code LOC (deterministic) and tokens/cost (API telemetry)."

providers:
  - id: anthropic:messages:claude-haiku-4-5-20251001
    config: { max_tokens: 8192, temperature: 1 }
  - id: anthropic:messages:claude-sonnet-4-6
    config: { max_tokens: 8192, temperature: 1 }
  - id: anthropic:messages:claude-opus-4-8
    config: { max_tokens: 8192, temperature: 1 }

prompts:
  - id: file://arms/baseline.js
    label: baseline (no skill)
  - id: file://arms/caveman.js
    label: caveman
  - id: file://arms/ponytail.js
    label: ponytail

defaultTest:
  assert:
    - type: javascript
      value: file://loc.js
      metric: code_loc
    - type: javascript
      value: file://correctness.js
      metric: correct

tests:
  - vars: { task: "Write me a Python function that validates email addresses." }
  - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
  - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
  - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
  - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }
