From 08440027f252680024210249c9c51084c8b3f4a5 Mon Sep 17 00:00:00 2001 From: Emeriko Date: Wed, 17 Jun 2026 04:31:52 +0200 Subject: [PATCH] fix(examples): replace hand-written examples with real benchmark output The examples/ before/after blocks were authored by hand, not produced by a model. Issue #127 correctly noted that nobody hand-rolls quicksort for "sort this array" - every model just calls .sort(). Regenerate all examples verbatim from a real benchmark run (Claude Haiku 4.5, no-skill arm vs ponytail arm, benchmarks/output.json) so the before/after is reproducible, not authored: email 75->3, debounce 116->10, csv 20->3, countdown 267->9, rate-limit 128->10 LOC - Delete sorting.md (pure strawman) plus the other hand-written caricatures (api-endpoint, caching, date-picker) - Add benchmarks/generate-examples.mjs to regenerate examples from any run - examples/README.md indexes the set and documents how to reproduce Closes #127 Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmarks/generate-examples.mjs | 63 +++++ examples/README.md | 17 ++ examples/api-endpoint.md | 68 ------ examples/caching.md | 74 ------ examples/csv-sum.md | 71 ++++++ examples/date-picker.md | 48 ---- examples/debounce.md | 211 +++++++++++++++++ examples/email-validation.md | 171 +++++++++++--- examples/rate-limit.md | 272 +++++++++++++++++++++ examples/react-countdown.md | 390 +++++++++++++++++++++++++++++++ examples/sorting.md | 42 ---- 11 files changed, 1162 insertions(+), 265 deletions(-) create mode 100644 benchmarks/generate-examples.mjs create mode 100644 examples/README.md delete mode 100644 examples/api-endpoint.md delete mode 100644 examples/caching.md create mode 100644 examples/csv-sum.md delete mode 100644 examples/date-picker.md create mode 100644 examples/debounce.md create mode 100644 examples/rate-limit.md create mode 100644 examples/react-countdown.md delete mode 100644 examples/sorting.md diff --git a/benchmarks/generate-examples.mjs b/benchmarks/generate-examples.mjs new file mode 100644 index 0000000..4cda716 --- /dev/null +++ b/benchmarks/generate-examples.mjs @@ -0,0 +1,63 @@ +// Generate examples/*.md verbatim from a real benchmark run (output.json): +// each file shows the same task answered with no skill vs with ponytail, same model. +// node benchmarks/generate-examples.mjs +import { readFileSync, writeFileSync } from 'node:fs'; +import loc from './loc.js'; + +const j = JSON.parse(readFileSync(new URL('./output.json', import.meta.url), 'utf8')); +const isHaiku = (id) => id.includes('haiku'); + +const meta = [ + [/validates email/, 'email-validation', 'Email Validation'], + [/debounce/, 'debounce', 'Debounce'], + [/sales\.csv/, 'csv-sum', 'CSV Sum'], + [/countdown timer/, 'react-countdown', 'Countdown Timer'], + [/rate limiting/, 'rate-limit', 'Rate Limiting'], +]; + +const pick = (re, armIdx) => + j.results.results.find((r) => isHaiku(r.provider.id) && r.promptIdx === armIdx && re.test(r.vars.task)); + +const rows = []; +for (const [re, slug, title] of meta) { + const b = pick(re, 0), p = pick(re, 2); + if (!b || !p) { console.log('MISS', slug, !!b, !!p); continue; } + const bL = loc(b.response.output).score, pL = loc(p.response.output).score; + const md = `# ${title} + +**Task:** "${b.vars.task}" + +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source \`benchmarks/output.json\`. Reproduce: \`npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml\`. + +## Without Ponytail — ${bL} lines of code + +${b.response.output.trim()} + +## With Ponytail — ${pL} lines of code + +${p.response.output.trim()} + +**${bL} → ${pL} lines of code** — same model, same prompt. +`; + writeFileSync(new URL(`../examples/${slug}.md`, import.meta.url), md); + rows.push([title, slug, bL, pL]); + console.log('wrote examples/' + slug + '.md', bL, '->', pL); +} + +const tbl = rows.map(([t, s, b, p]) => `| [${t}](${s}.md) | ${b} | ${p} |`).join('\n'); +const readme = `# Examples + +Real model output, verbatim from benchmark runs — the same task answered by the same model +with no skill (\`## Without Ponytail\`) and with ponytail (\`## With Ponytail\`), so you can +compare side by side. Model: Claude Haiku 4.5, temperature 1, source \`benchmarks/output.json\`. + +These are not hand-written. Reproduce them yourself: +\`npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml\`. Method, all three models, and +median-of-10 numbers: [../benchmarks/](../benchmarks/). + +| Example | Without (LOC) | With (LOC) | +|---|--:|--:| +${tbl} +`; +writeFileSync(new URL('../examples/README.md', import.meta.url), readme); +console.log('wrote examples/README.md'); diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..4dacc4e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,17 @@ +# Examples + +Real model output, verbatim from benchmark runs — the same task answered by the same model +with no skill (`## Without Ponytail`) and with ponytail (`## With Ponytail`), so you can +compare side by side. Model: Claude Haiku 4.5, temperature 1, source `benchmarks/output.json`. + +These are not hand-written. Reproduce them yourself: +`npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. Method, all three models, and +median-of-10 numbers: [../benchmarks/](../benchmarks/). + +| Example | Without (LOC) | With (LOC) | +|---|--:|--:| +| [Email Validation](email-validation.md) | 75 | 3 | +| [Debounce](debounce.md) | 116 | 10 | +| [CSV Sum](csv-sum.md) | 20 | 3 | +| [Countdown Timer](react-countdown.md) | 267 | 9 | +| [Rate Limiting](rate-limit.md) | 128 | 10 | diff --git a/examples/api-endpoint.md b/examples/api-endpoint.md deleted file mode 100644 index 4e26570..0000000 --- a/examples/api-endpoint.md +++ /dev/null @@ -1,68 +0,0 @@ -# API Endpoint - -**Task:** "Add an endpoint that returns a user by id." - -## Without Ponytail - -``` -app/ -├── controllers/user_controller.py -├── services/user_service.py -├── repositories/user_repository.py -├── schemas/user_schemas.py -└── exceptions/user_exceptions.py -``` - -```python -# schemas/user_schemas.py -class UserResponseDTO(BaseModel): - id: int - name: str - email: str - -# repositories/user_repository.py -class UserRepository: - def __init__(self, db: Session): - self.db = db - def find_by_id(self, user_id: int) -> User | None: - return self.db.get(User, user_id) - -# services/user_service.py -class UserService: - def __init__(self, repo: UserRepository): - self.repo = repo - def get_user(self, user_id: int) -> User: - user = self.repo.find_by_id(user_id) - if user is None: - raise UserNotFoundError(user_id) - return user - -# controllers/user_controller.py -@router.get("/users/{user_id}", response_model=UserResponseDTO) -def get_user(user_id: int, service: UserService = Depends(get_user_service)): - try: - return service.get_user(user_id) - except UserNotFoundError: - raise HTTPException(status_code=404, detail="User not found") -``` - -Five files, three classes, a custom exception, and a dependency-injection chain, wrapping one database call. - -## With Ponytail - -```python -# ponytail: drop the layers; keep the response schema, it whitelists what leaves the API -class UserOut(BaseModel): - id: int - name: str - email: str - -@app.get("/users/{user_id}", response_model=UserOut) -def get_user(user_id: int, db: Session = Depends(get_db)): - user = db.get(User, user_id) - if not user: - raise HTTPException(404) - return user -``` - -**5 files → 9 lines.** The repository, service, and custom exception were ceremony. The response schema was not: it whitelists which fields leave the API, so it stays. Returning the raw ORM model (`return user`) would leak every column, including the ones you never meant to expose. That is the line ponytail draws, and it is the same one the skill draws in "when NOT to be lazy": cut the layers, keep the trust boundary. Add a service layer when a second caller shows up, if it ever does. diff --git a/examples/caching.md b/examples/caching.md deleted file mode 100644 index 7dbe205..0000000 --- a/examples/caching.md +++ /dev/null @@ -1,74 +0,0 @@ -# Caching System - -**Task:** "We should cache these API responses." - -## Without Ponytail - -```python -import time -import threading -from collections import OrderedDict -from dataclasses import dataclass -from typing import Any, Optional - - -@dataclass -class CacheEntry: - value: Any - expires_at: float - - -class TTLCache: - """Thread-safe LRU cache with per-entry TTL and max size.""" - - def __init__(self, max_size: int = 1000, default_ttl: float = 300.0): - self._store: OrderedDict[str, CacheEntry] = OrderedDict() - self._lock = threading.Lock() - self.max_size = max_size - self.default_ttl = default_ttl - self.hits = 0 - self.misses = 0 - - def get(self, key: str) -> Optional[Any]: - with self._lock: - entry = self._store.get(key) - if entry is None or entry.expires_at < time.monotonic(): - self.misses += 1 - self._store.pop(key, None) - return None - self._store.move_to_end(key) - self.hits += 1 - return entry.value - - def set(self, key: str, value: Any, ttl: Optional[float] = None) -> None: - with self._lock: - if len(self._store) >= self.max_size: - self._store.popitem(last=False) - self._store[key] = CacheEntry( - value, time.monotonic() + (ttl or self.default_ttl) - ) - - # ... plus invalidation, stats endpoint, and unit tests for all of it -``` - -120 lines of custom infrastructure for a problem nobody has measured yet. - -## With Ponytail - -First question: **do you actually need a cache?** - -- **Unsure?** Ship without it. Add it when you measure the problem. (YAGNI) -- **Pure function, hot path?** The standard library has it: - - ```python - # ponytail: stdlib covers this - from functools import lru_cache - - @lru_cache(maxsize=1000) - def fetch(key): ... - ``` - -- **Real distributed caching needs?** Use Redis / memcached / your platform's - cache. Infrastructure problems get infrastructure, not a homemade class. - -**120 lines → 0–3 lines.** The fastest cache is the one you didn't have to debug. diff --git a/examples/csv-sum.md b/examples/csv-sum.md new file mode 100644 index 0000000..6560688 --- /dev/null +++ b/examples/csv-sum.md @@ -0,0 +1,71 @@ +# CSV Sum + +**Task:** "Write Python code that reads sales.csv and sums the 'amount' column." + +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail — 20 lines of code + +# Python code to read sales.csv and sum the 'amount' column + +```python +import pandas as pd + +# Read the CSV file +df = pd.read_csv('sales.csv') + +# Sum the 'amount' column +total_amount = df['amount'].sum() + +# Print the result +print(f"Total amount: ${total_amount:,.2f}") +``` + +## Alternative methods: + +**Using CSV module (without pandas):** +```python +import csv + +total_amount = 0 + +with open('sales.csv', 'r') as file: + reader = csv.DictReader(file) + for row in reader: + total_amount += float(row['amount']) + +print(f"Total amount: ${total_amount:,.2f}") +``` + +**Using pandas with error handling:** +```python +import pandas as pd + +try: + df = pd.read_csv('sales.csv') + total_amount = df['amount'].sum() + print(f"Total amount: ${total_amount:,.2f}") +except FileNotFoundError: + print("Error: sales.csv not found") +except KeyError: + print("Error: 'amount' column not found") +``` + +**The pandas method is recommended** because it's: +- More concise and readable +- Handles data types automatically +- Provides built-in error handling +- Efficient for large files + +## With Ponytail — 3 lines of code + +```python +import csv + +total = sum(float(row['amount']) for row in csv.DictReader(open('sales.csv'))) +print(total) +``` + +Skipped: pandas, error handling, file closing — add when the CSV is large, malformed, or you need more analysis. + +**20 → 3 lines of code** — same model, same prompt. diff --git a/examples/date-picker.md b/examples/date-picker.md deleted file mode 100644 index c357fb7..0000000 --- a/examples/date-picker.md +++ /dev/null @@ -1,48 +0,0 @@ -# Date Picker - -**Task:** "Add a date picker to this form." - -## Without Ponytail - -```bash -npm install flatpickr -``` - -```jsx -import flatpickr from "flatpickr"; -import "flatpickr/dist/flatpickr.min.css"; -import { useEffect, useRef } from "react"; - -export default function DatePicker({ value, onChange, minDate, maxDate }) { - const inputRef = useRef(null); - const instanceRef = useRef(null); - - useEffect(() => { - instanceRef.current = flatpickr(inputRef.current, { - defaultDate: value, - minDate, - maxDate, - dateFormat: "Y-m-d", - onChange: ([date]) => onChange(date), - }); - return () => instanceRef.current?.destroy(); - }, []); - - useEffect(() => { - instanceRef.current?.setDate(value, false); - }, [value]); - - return ; -} -``` - -One dependency, one wrapper component, two `useEffect` hooks, a cleanup function, and a CSS import, to pick a date. - -## With Ponytail - -```html - - -``` - -**1 dependency + 30 lines → 0 dependencies + 1 line.** Native, accessible, localized, keyboard-navigable, mobile-friendly. The browser team already did the work. diff --git a/examples/debounce.md b/examples/debounce.md new file mode 100644 index 0000000..829017c --- /dev/null +++ b/examples/debounce.md @@ -0,0 +1,211 @@ +# Debounce + +**Task:** "Add debounce to a search input in vanilla JavaScript. It currently fires an API call on every keystroke." + +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail — 116 lines of code + +# Debounce Search Input + +Here's a practical implementation to prevent excessive API calls: + +## Basic Debounce Function + +```javascript +function debounce(func, delay) { + let timeoutId; + return function (...args) { + clearTimeout(timeoutId); + timeoutId = setTimeout(() => func(...args), delay); + }; +} + +// Usage +const searchInput = document.getElementById('search'); +const resultsContainer = document.getElementById('results'); + +const handleSearch = async (query) => { + if (!query.trim()) { + resultsContainer.innerHTML = ''; + return; + } + + try { + const response = await fetch(`/api/search?q=${encodeURIComponent(query)}`); + const data = await response.json(); + displayResults(data); + } catch (error) { + console.error('Search failed:', error); + } +}; + +// Debounce the search function (300ms delay) +const debouncedSearch = debounce(handleSearch, 300); + +searchInput.addEventListener('input', (e) => { + debouncedSearch(e.target.value); +}); +``` + +## Enhanced Version with Loading State + +```javascript +const searchInput = document.getElementById('search'); +const resultsContainer = document.getElementById('results'); +const loadingIndicator = document.getElementById('loading'); + +const handleSearch = async (query) => { + if (!query.trim()) { + resultsContainer.innerHTML = ''; + loadingIndicator.style.display = 'none'; + return; + } + + loadingIndicator.style.display = 'block'; + + try { + const response = await fetch(`/api/search?q=${encodeURIComponent(query)}`); + if (!response.ok) throw new Error('Search failed'); + + const data = await response.json(); + displayResults(data); + } catch (error) { + resultsContainer.innerHTML = '

Search failed. Try again.

'; + console.error('Search error:', error); + } finally { + loadingIndicator.style.display = 'none'; + } +}; + +const debouncedSearch = debounce(handleSearch, 300); + +searchInput.addEventListener('input', (e) => { + debouncedSearch(e.target.value); +}); + +function displayResults(data) { + resultsContainer.innerHTML = data + .map(item => `
${item.title}
`) + .join(''); +} +``` + +## Advanced: Debounce with Cancel & Immediate Options + +```javascript +function debounce(func, delay, options = {}) { + let timeoutId; + let lastArgs; + + const debounced = function (...args) { + lastArgs = args; + clearTimeout(timeoutId); + + if (options.immediate && !timeoutId) { + func(...args); + } + + timeoutId = setTimeout(() => { + if (!options.immediate) { + func(...lastArgs); + } + timeoutId = null; + }, delay); + }; + + // Allow manual cancellation + debounced.cancel = () => clearTimeout(timeoutId); + + return debounced; +} + +// Usage with options +const debouncedSearch = debounce(handleSearch, 300, { immediate: false }); + +// Cancel pending search if needed +searchInput.addEventListener('blur', () => { + debouncedSearch.cancel(); +}); +``` + +## HTML Example + +```html +
+ + +
+
+ + +``` + +## Key Benefits + +| Feature | Benefit | +|---------|---------| +| **Debounce** | Reduces API calls from 100+ to 1-2 per search | +| **300ms delay** | Balances responsiveness with performance | +| **Loading state** | Provides user feedback | +| **Error handling** | Graceful failure management | +| **Cancel method** | Cleanup when needed | + +**Typical keystroke reduction:** 10 characters = 10 API calls → 1-2 API calls with debounce ✅ + +## With Ponytail — 10 lines of code + +```javascript +const searchInput = document.querySelector('input[type="search"]'); +let debounceTimer; + +searchInput.addEventListener('input', (e) => { + clearTimeout(debounceTimer); + debounceTimer = setTimeout(() => { + fetch(`/api/search?q=${encodeURIComponent(e.target.value)}`) + .then(r => r.json()) + .then(data => console.log(data)); // replace with your render + }, 300); +}); +``` + +Skipped: debounce utility function, class wrapper, config object — `setTimeout` + `clearTimeout` is the debounce. Add a utility when you need it on 3+ inputs. + +**116 → 10 lines of code** — same model, same prompt. diff --git a/examples/email-validation.md b/examples/email-validation.md index 4b69aa6..eb71739 100644 --- a/examples/email-validation.md +++ b/examples/email-validation.md @@ -1,51 +1,156 @@ # Email Validation -**Task:** "Validate an email address in Python." +**Task:** "Write me a Python function that validates email addresses." -## Without Ponytail +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail — 75 lines of code + +# Email Validation Function + +Here's a practical Python function that validates email addresses: ```python import re -EMAIL_PATTERN = re.compile( - r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' -) - -class EmailValidator: - """Validates email addresses against RFC-like rules.""" - - def __init__(self, pattern: re.Pattern = EMAIL_PATTERN): - self.pattern = pattern - - def validate(self, email: str) -> bool: - if not isinstance(email, str): - raise TypeError("email must be a string") - email = email.strip() - if not email: - return False - return bool(self.pattern.match(email)) - - def validate_email(email: str) -> bool: - """Convenience wrapper around EmailValidator.""" - return EmailValidator().validate(email) + """ + Validates an email address using a regular expression. + + Args: + email (str): The email address to validate + + Returns: + bool: True if valid, False otherwise + """ + # Regular expression pattern for email validation + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + + return re.match(pattern, email) is not None + + +# Test cases +if __name__ == "__main__": + test_emails = [ + "user@example.com", # Valid + "john.doe@company.co.uk", # Valid + "test+tag@domain.org", # Valid + "invalid.email@", # Invalid + "missing@domain", # Invalid + "@nodomain.com", # Invalid + "spaces in@email.com", # Invalid + "user@domain..com", # Invalid (double dot) + ] + + for email in test_emails: + result = validate_email(email) + print(f"{email:<30} -> {result}") ``` -A class, a wrapper, a regex that still rejects valid addresses and accepts invalid ones. Regex cannot validate email. Only a delivery attempt can. - -## With Ponytail +## More Robust Version (with additional checks) ```python -# ponytail: good enough, real validation is sending the mail -"@" in email and "." in email.split("@")[-1] +import re + +def validate_email_advanced(email: str) -> tuple[bool, str]: + """ + Advanced email validation with detailed feedback. + + Args: + email (str): The email address to validate + + Returns: + tuple: (is_valid, message) + """ + # Basic checks + if not email or not isinstance(email, str): + return False, "Email must be a non-empty string" + + email = email.strip() + + if len(email) > 254: + return False, "Email is too long (max 254 characters)" + + # Regular expression pattern + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + + if not re.match(pattern, email): + return False, "Email format is invalid" + + # Additional checks + local_part, domain = email.rsplit('@', 1) + + if len(local_part) > 64: + return False, "Local part is too long (max 64 characters)" + + if local_part.startswith('.') or local_part.endswith('.'): + return False, "Local part cannot start or end with a dot" + + if '..' in email: + return False, "Email cannot contain consecutive dots" + + return True, "Email is valid" + + +# Test the advanced version +if __name__ == "__main__": + test_emails = [ + "user@example.com", + "invalid..email@domain.com", + ".invalid@domain.com", + "user@domain", + ] + + for email in test_emails: + is_valid, message = validate_email_advanced(email) + print(f"{email:<35} -> {is_valid:5} ({message})") ``` -Or, if it must be thorough, the standard library has it: +## Using a Third-Party Library (Recommended for Production) ```python -# ponytail: stdlib covers this -from email.utils import parseaddr -"@" in parseaddr(email)[1] +from email_validator import validate_email, EmailNotValidError + +def validate_email_production(email: str) -> bool: + """ + Validates email using the email_validator library. + More reliable for production use. + + Install with: pip install email-validator + """ + try: + # Validate and normalize the email + valid = validate_email(email) + return True + except EmailNotValidError: + return False + + +# Usage +if __name__ == "__main__": + print(validate_email_production("user@example.com")) # True + print(validate_email_production("invalid@")) # False ``` -**27 lines → 1 line.** And the honest answer: let the confirmation email reject it. That's what confirmation emails are for. +## Comparison + +| Method | Pros | Cons | +|--------|------|------| +| **Regex (Simple)** | Fast, no dependencies | Limited validation | +| **Regex (Advanced)** | More thorough | Still not RFC-compliant | +| **email-validator** | RFC-compliant, reliable | External dependency | + +**Recommendation**: Use the `email-validator` library for production applications, as it properly handles RFC 5321/5322 standards. Use the simple regex version for quick scripts or when you want to avoid dependencies. + +## With Ponytail — 3 lines of code + +```python +import re + +def is_valid_email(email: str) -> bool: + return bool(re.match(r'^[^@]+@[^@]+\.[^@]+$', email)) +``` + +Skipped: RFC 5322 parser, DNS MX lookup, confirmation email. Add when you actually need to reject `user+tag@sub.domain.co.uk` or catch typos — until then, this catches 99% of "oops I fat-fingered it" cases. + +**75 → 3 lines of code** — same model, same prompt. diff --git a/examples/rate-limit.md b/examples/rate-limit.md new file mode 100644 index 0000000..84ebe56 --- /dev/null +++ b/examples/rate-limit.md @@ -0,0 +1,272 @@ +# Rate Limiting + +**Task:** "Add rate limiting to my FastAPI endpoint so users can't spam it." + +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail — 128 lines of code + +# Rate Limiting in FastAPI + +Here are several approaches, from simplest to most robust: + +## 1. **Using `slowapi` (Recommended - Easiest)** + +```bash +pip install slowapi +``` + +```python +from fastapi import FastAPI, Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from fastapi.responses import JSONResponse + +app = FastAPI() + +limiter = Limiter(key_func=get_remote_address) +app.state.limiter = limiter +app.add_exception_handler(RateLimitExceeded, lambda request, exc: JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded"} +)) + +@app.get("/api/data") +@limiter.limit("5/minute") +async def get_data(request: Request): + return {"message": "Success"} +``` + +**Rate limit formats:** +- `"5/minute"` - 5 requests per minute +- `"100/hour"` - 100 requests per hour +- `"1000/day"` - 1000 requests per day + +--- + +## 2. **Using `limits` Library (More Control)** + +```bash +pip install limits +``` + +```python +from fastapi import FastAPI, HTTPException, Request +from limits import parse +from limits.storage import MemoryStorage +from limits.strategies import MovingWindowRateLimiter + +app = FastAPI() + +storage = MemoryStorage() +limiter = MovingWindowRateLimiter(storage) +rate_limit = parse("5/minute") + +@app.get("/api/data") +async def get_data(request: Request): + client_id = request.client.host + + if not limiter.hit(rate_limit, client_id): + raise HTTPException(status_code=429, detail="Rate limit exceeded") + + return {"message": "Success"} +``` + +--- + +## 3. **Custom Middleware (Full Control)** + +```python +from fastapi import FastAPI, HTTPException, Request +from datetime import datetime, timedelta +from collections import defaultdict +import asyncio + +app = FastAPI() + +# Store request timestamps per IP +request_history = defaultdict(list) +RATE_LIMIT = 5 # requests +TIME_WINDOW = 60 # seconds + +@app.middleware("http") +async def rate_limit_middleware(request: Request, call_next): + client_ip = request.client.host + now = datetime.now() + + # Clean old requests outside time window + request_history[client_ip] = [ + req_time for req_time in request_history[client_ip] + if now - req_time < timedelta(seconds=TIME_WINDOW) + ] + + # Check if limit exceeded + if len(request_history[client_ip]) >= RATE_LIMIT: + return JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded"} + ) + + # Record this request + request_history[client_ip].append(now) + + return await call_next(request) + +@app.get("/api/data") +async def get_data(): + return {"message": "Success"} +``` + +--- + +## 4. **Per-User Rate Limiting (With Authentication)** + +```python +from fastapi import FastAPI, Depends, HTTPException +from slowapi import Limiter +from slowapi.util import get_remote_address + +app = FastAPI() +limiter = Limiter(key_func=get_remote_address) + +def get_user_id(token: str = Header(None)) -> str: + # Your auth logic here + return token or "anonymous" + +@app.get("/api/data") +@limiter.limit("10/minute") +async def get_data(request: Request, user_id: str = Depends(get_user_id)): + return {"message": f"Success for {user_id}"} +``` + +--- + +## 5. **Redis-Based Rate Limiting (Production)** + +```bash +pip install slowapi redis +``` + +```python +from fastapi import FastAPI, Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from slowapi.storage import RedisStorage +from redis import Redis +from fastapi.responses import JSONResponse + +redis_client = Redis.from_url("redis://localhost:6379") +storage = RedisStorage(redis_client) +limiter = Limiter(key_func=get_remote_address, storage=storage) + +app = FastAPI() +app.state.limiter = limiter + +app.add_exception_handler( + RateLimitExceeded, + lambda request, exc: JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded"} + ) +) + +@app.get("/api/data") +@limiter.limit("5/minute") +async def get_data(request: Request): + return {"message": "Success"} +``` + +--- + +## 6. **Complete Example with Multiple Endpoints** + +```python +from fastapi import FastAPI, Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from fastapi.responses import JSONResponse + +app = FastAPI() +limiter = Limiter(key_func=get_remote_address) +app.state.limiter = limiter + +@app.exception_handler(RateLimitExceeded) +async def rate_limit_handler(request: Request, exc: RateLimitExceeded): + return JSONResponse( + status_code=429, + content={ + "detail": "Too many requests", + "retry_after": exc.detail + } + ) + +# Strict limit +@app.post("/api/login") +@limiter.limit("3/minute") +async def login(request: Request): + return {"token": "xxx"} + +# Moderate limit +@app.get("/api/data") +@limiter.limit("30/minute") +async def get_data(request: Request): + return {"data": []} + +# Generous limit +@app.get("/api/public") +@limiter.limit("100/minute") +async def get_public(request: Request): + return {"public": "data"} +``` + +--- + +## Comparison Table + +| Method | Pros | Cons | Best For | +|--------|------|------|----------| +| **slowapi** | Easy, decorator-based | In-memory only | Development, small apps | +| **limits** | Flexible, multiple backends | More setup | Custom logic | +| **Middleware** | Full control, no dependencies | Manual cleanup | Learning, simple cases | +| **Redis** | Distributed, scalable | Requires Redis | Production, multi-server | + +--- + +## Testing Rate Limits + +```python +import httpx + +async def test_rate_limit(): + async with httpx.AsyncClient(app=app, base_url="http://test") as client: + for i in range(6): + response = await client.get("/api/data") + print(f"Request {i+1}: {response.status_code}") + # First 5 should be 200, 6th should be 429 +``` + +**Recommendation:** Use `slowapi` for most cases—it's the most popular and easiest to implement! 🚀 + +## With Ponytail — 10 lines of code + +```python +from fastapi import FastAPI, HTTPException +from slowapi import Limiter +from slowapi.util import get_remote_address + +app = FastAPI() +limiter = Limiter(key_func=get_remote_address) +app.state.limiter = limiter + +@app.get("/api/endpoint") +@limiter.limit("10/minute") +async def my_endpoint(request): + return {"status": "ok"} +``` + +Skipped: custom rate limit logic, Redis, sliding windows — `slowapi` handles it. Add when: you need distributed rate limiting across multiple servers (swap `Limiter` for Redis backend) or per-user limits (add `key_func=lambda r: r.headers.get("authorization")`). + +**128 → 10 lines of code** — same model, same prompt. diff --git a/examples/react-countdown.md b/examples/react-countdown.md new file mode 100644 index 0000000..5bfc8b9 --- /dev/null +++ b/examples/react-countdown.md @@ -0,0 +1,390 @@ +# Countdown Timer + +**Task:** "Build me a countdown timer component in React that counts down from a given number of seconds." + +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail — 267 lines of code + +# React Countdown Timer Component + +Here's a complete countdown timer component with multiple variations: + +## Basic Countdown Timer + +```jsx +import { useState, useEffect } from 'react'; + +export function CountdownTimer({ initialSeconds = 60 }) { + const [seconds, setSeconds] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && seconds > 0) { + interval = setInterval(() => { + setSeconds(seconds => seconds - 1); + }, 1000); + } else if (seconds === 0) { + setIsActive(false); + } + + return () => clearInterval(interval); + }, [isActive, seconds]); + + const toggle = () => setIsActive(!isActive); + const reset = () => { + setSeconds(initialSeconds); + setIsActive(false); + }; + + return ( +
+
{seconds}s
+
+ + +
+
+ ); +} +``` + +## Advanced Timer with Formatted Display + +```jsx +import { useState, useEffect } from 'react'; + +export function AdvancedCountdownTimer({ + initialSeconds = 3600, + onComplete = () => {}, + showMilliseconds = false +}) { + const [timeLeft, setTimeLeft] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && timeLeft > 0) { + interval = setInterval(() => { + setTimeLeft(prev => { + if (prev <= 1) { + setIsActive(false); + onComplete(); + return 0; + } + return prev - 1; + }); + }, 1000); + } + + return () => clearInterval(interval); + }, [isActive, timeLeft, onComplete]); + + const formatTime = (totalSeconds) => { + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + const parts = []; + if (hours > 0) parts.push(String(hours).padStart(2, '0')); + parts.push(String(minutes).padStart(2, '0')); + parts.push(String(seconds).padStart(2, '0')); + + return parts.join(':'); + }; + + const toggle = () => setIsActive(!isActive); + const reset = () => { + setTimeLeft(initialSeconds); + setIsActive(false); + }; + + const progressPercent = ((initialSeconds - timeLeft) / initialSeconds) * 100; + + return ( +
+
+ {formatTime(timeLeft)} +
+
+
+
+
+ + +
+
+ ); +} +``` + +## Custom Hook Version + +```jsx +import { useState, useEffect, useCallback } from 'react'; + +export function useCountdown(initialSeconds) { + const [seconds, setSeconds] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && seconds > 0) { + interval = setInterval(() => { + setSeconds(prev => prev - 1); + }, 1000); + } else if (seconds === 0 && isActive) { + setIsActive(false); + } + + return () => clearInterval(interval); + }, [isActive, seconds]); + + const start = useCallback(() => setIsActive(true), []); + const pause = useCallback(() => setIsActive(false), []); + const reset = useCallback(() => { + setSeconds(initialSeconds); + setIsActive(false); + }, [initialSeconds]); + + return { + seconds, + isActive, + start, + pause, + reset, + setSeconds + }; +} + +// Usage +export function TimerWithHook() { + const { seconds, isActive, start, pause, reset } = useCountdown(60); + + return ( +
+
{seconds}s
+ + +
+ ); +} +``` + +## Styled Component with Animations + +```jsx +import { useState, useEffect } from 'react'; +import styled from 'styled-components'; + +const TimerContainer = styled.div` + display: flex; + flex-direction: column; + align-items: center; + gap: 20px; + padding: 30px; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + border-radius: 15px; + color: white; + font-family: 'Arial', sans-serif; +`; + +const Display = styled.div` + font-size: 48px; + font-weight: bold; + font-variant-numeric: tabular-nums; + animation: ${props => props.isWarning ? 'pulse 0.5s infinite' : 'none'}; + + @keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.5; } + } +`; + +const Controls = styled.div` + display: flex; + gap: 10px; +`; + +const Button = styled.button` + padding: 10px 20px; + font-size: 16px; + border: none; + border-radius: 8px; + cursor: pointer; + background-color: ${props => props.variant === 'danger' ? '#ff6b6b' : '#4ecdc4'}; + color: white; + transition: all 0.3s ease; + + &:hover { + transform: translateY(-2px); + box-shadow: 0 5px 15px rgba(0, 0, 0, 0.2); + } + + &:active { + transform: translateY(0); + } +`; + +export function StyledCountdownTimer({ initialSeconds = 60 }) { + const [seconds, setSeconds] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && seconds > 0) { + interval = setInterval(() => { + setSeconds(prev => prev - 1); + }, 1000); + } else if (seconds === 0) { + setIsActive(false); + } + + return () => clearInterval(interval); + }, [isActive, seconds]); + + const isWarning = seconds <= 10 && seconds > 0; + + return ( + + {seconds}s + + + + + + ); +} +``` + +## CSS Styling + +```css +.countdown-timer { + display: flex; + flex-direction: column; + align-items: center; + gap: 20px; + padding: 20px; + border: 2px solid #333; + border-radius: 10px; + background-color: #f5f5f5; +} + +.timer-display { + font-size: 48px; + font-weight: bold; + font-variant-numeric: tabular-nums; + color: #333; +} + +.timer-controls { + display: flex; + gap: 10px; +} + +.timer-controls button { + padding: 10px 20px; + font-size: 16px; + border: none; + border-radius: 5px; + cursor: pointer; + background-color: #667eea; + color: white; + transition: background-color 0.3s; +} + +.timer-controls button:hover { + background-color: #5568d3; +} + +.progress-bar { + width: 100%; + height: 8px; + background-color: #e0e0e0; + border-radius: 4px; + overflow: hidden; +} + +.progress-fill { + height: 100%; + background-color: #667eea; + transition: width 0.3s linear; +} +``` + +## Usage Examples + +```jsx +// Basic usage + + +// Advanced with callback + alert('Time\'s up!')} +/> + +// Using the hook + + +// Styled version + +``` + +## Features + +✅ **Basic Timer** - Simple countdown with start/pause/reset +✅ **Advanced Timer** - Formatted display (HH:MM:SS), progress bar, callbacks +✅ **Custom Hook** - Reusable logic for any component +✅ **Styled Components** - Beautiful UI with animations +✅ **Warning State** - Visual feedback when time is running out +✅ **Completion Callback** - Execute code when timer finishes + +Choose the version that best fits your needs! + +## With Ponytail — 9 lines of code + +```jsx +export function CountdownTimer({ seconds }) { + const [remaining, setRemaining] = React.useState(seconds); + + React.useEffect(() => { + if (remaining <= 0) return; + const timer = setInterval(() => setRemaining(r => r - 1), 1000); + return () => clearInterval(timer); + }, [remaining]); + + return
{remaining}s
; +} +``` + +Skipped: pause/resume, formatted display (mm:ss), sound on zero, styling — add when needed. + +**267 → 9 lines of code** — same model, same prompt. diff --git a/examples/sorting.md b/examples/sorting.md deleted file mode 100644 index aa31f3b..0000000 --- a/examples/sorting.md +++ /dev/null @@ -1,42 +0,0 @@ -# Sorting - -**Task:** "Sort this array of numbers." - -## Without Ponytail - -```js -function quickSort(arr, compareFn = (a, b) => a - b) { - if (arr.length <= 1) return arr; - - const pivot = arr[Math.floor(arr.length / 2)]; - const left = []; - const right = []; - const equal = []; - - for (const item of arr) { - const cmp = compareFn(item, pivot); - if (cmp < 0) left.push(item); - else if (cmp > 0) right.push(item); - else equal.push(item); - } - - return [ - ...quickSort(left, compareFn), - ...equal, - ...quickSort(right, compareFn), - ]; -} - -const sorted = quickSort(numbers); -``` - -A hand-rolled quicksort. It allocates three arrays per recursion level, blows the stack on large inputs, and replaces an engine-optimized native sort with homework. - -## With Ponytail - -```js -// ponytail: this exists -numbers.sort((a, b) => a - b) -``` - -**24 lines → 1 line.** Every runtime ships a sort tuned by people whose whole job is sorting. Use it.