Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce153bc95f | ||
|
|
084f10fb48 | ||
|
|
2e6a93765a | ||
|
|
386f95734a | ||
|
|
60a75f8159 | ||
|
|
b41cb8d3af | ||
|
|
d676635325 | ||
|
|
f02f9424a5 | ||
|
|
2302fbc843 | ||
|
|
c1c80f3cc8 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ponytail",
|
||||
"version": "4.4.0",
|
||||
"version": "4.6.0",
|
||||
"description": "Lazy senior dev mode. Forces the simplest, shortest solution that actually works: YAGNI, stdlib first, no unrequested abstractions.",
|
||||
"author": {
|
||||
"name": "Dietrich Gebert",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ponytail",
|
||||
"version": "4.4.0",
|
||||
"version": "4.6.0",
|
||||
"description": "Lazy senior dev mode. Forces the simplest, shortest solution that actually works: YAGNI, stdlib first, no unrequested abstractions.",
|
||||
"author": {
|
||||
"name": "Dietrich Gebert",
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "ponytail",
|
||||
"description": "Lazy senior dev mode for AI agents. The best code is the code you never wrote.",
|
||||
"owner": {
|
||||
"name": "Dietrich Gebert",
|
||||
"url": "https://github.com/DietrichGebert"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "ponytail",
|
||||
"description": "Forces the laziest solution that works. YAGNI, stdlib first, one line over fifty.",
|
||||
"source": "./",
|
||||
"category": "productivity",
|
||||
"tags": ["yagni", "minimalism", "code-review", "productivity"],
|
||||
"commands": "commands/",
|
||||
"skills": "skills/",
|
||||
"hooks": "hooks/copilot-hooks.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "ponytail",
|
||||
"description": "Lazy senior dev mode. Forces the simplest, shortest solution that actually works: YAGNI, stdlib first, no unrequested abstractions.",
|
||||
"version": "4.6.0",
|
||||
"author": {
|
||||
"name": "Dietrich Gebert",
|
||||
"url": "https://github.com/DietrichGebert"
|
||||
},
|
||||
"homepage": "https://github.com/DietrichGebert/ponytail",
|
||||
"repository": "https://github.com/DietrichGebert/ponytail",
|
||||
"license": "MIT",
|
||||
"keywords": ["yagni", "minimalism", "code-review", "productivity"],
|
||||
"commands": "commands/",
|
||||
"skills": "skills/",
|
||||
"hooks": "hooks/copilot-hooks.json"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
name: test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install Python deps for correctness checks
|
||||
run: pip install pandas
|
||||
|
||||
- name: Check rule copies
|
||||
run: node scripts/check-rule-copies.js
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
@@ -9,6 +9,10 @@ node_modules/
|
||||
# promptfoo eval artifacts
|
||||
.promptfoo/
|
||||
benchmarks/output*
|
||||
benchmarks/benchmark-local-results.json
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
|
||||
# one-off social/announcement art, not repo content
|
||||
announce-*.png
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
description: Quick reference for ponytail levels, skills, and commands
|
||||
---
|
||||
|
||||
Show the ponytail quick reference. One shot, change nothing: do not switch mode, write flag files, or persist anything. Levels: /ponytail lite (build what's asked, name the lazier alternative in one line), /ponytail (full, the default ladder: YAGNI then stdlib then native then one line then minimum), /ponytail ultra (deletion before addition, challenges the requirement before building). Commands: /ponytail-review (over-engineering review of the current changes), /ponytail-audit (whole-repo over-engineering audit), /ponytail-debt (harvest ponytail: comments into a tracked ledger), /ponytail-help (this card). Deactivate with 'stop ponytail', 'normal mode', or /ponytail off; resume anytime with /ponytail. Default mode is full; change it with the PONYTAIL_DEFAULT_MODE environment variable (off|lite|full|ultra) or a config file at ~/.config/ponytail/config.json (Windows: %APPDATA%\ponytail\config.json) with {"defaultMode": "lite"}. Resolution order: env var, then config file, then full.
|
||||
@@ -14,7 +14,7 @@
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/github/stars/DietrichGebert/ponytail?style=flat-square&color=111111&label=stars" alt="Stars">
|
||||
<img src="https://img.shields.io/github/v/release/DietrichGebert/ponytail?style=flat-square&color=111111&label=release" alt="Release">
|
||||
<img src="https://img.shields.io/badge/works%20with-11%20agents-111111?style=flat-square" alt="Works with 11 agents">
|
||||
<img src="https://img.shields.io/badge/works%20with-13%20agents-111111?style=flat-square" alt="Works with 13 agents">
|
||||
<img src="https://img.shields.io/badge/license-MIT-111111?style=flat-square" alt="MIT license">
|
||||
</p>
|
||||
|
||||
@@ -71,6 +71,8 @@ Lazy, not negligent: trust-boundary validation, data-loss handling, security, an
|
||||
|
||||
The most effort ponytail will ever ask of you:
|
||||
|
||||
The Claude Code and Codex plugins run two tiny Node.js lifecycle hooks, so `node` needs to be on your PATH (note for Nix/nvm users: it must be on the non-interactive shell's PATH). If it isn't, the skills still work, the always-on activation just stays quiet instead of erroring on every prompt.
|
||||
|
||||
### Claude Code
|
||||
|
||||
```
|
||||
@@ -88,6 +90,29 @@ codex
|
||||
Open `/plugins`, select the Ponytail marketplace, and install Ponytail. Then
|
||||
open `/hooks`, review and trust its two lifecycle hooks, and start a new thread.
|
||||
|
||||
This same install also covers the Codex desktop app: restart the app after installing and it picks up the plugin.
|
||||
|
||||
### GitHub Copilot CLI
|
||||
|
||||
```bash
|
||||
copilot plugin marketplace add DietrichGebert/ponytail
|
||||
copilot plugin install ponytail@ponytail
|
||||
```
|
||||
|
||||
In an interactive Copilot CLI session, use the slash equivalents:
|
||||
|
||||
```
|
||||
/plugin marketplace add DietrichGebert/ponytail
|
||||
/plugin install ponytail@ponytail
|
||||
```
|
||||
|
||||
Copilot CLI namespaces plugin commands by plugin name. For example:
|
||||
|
||||
```text
|
||||
/ponytail:ponytail ultra
|
||||
/ponytail:ponytail-review
|
||||
```
|
||||
|
||||
### Pi agent harness
|
||||
|
||||
```
|
||||
@@ -116,11 +141,11 @@ That was it. He'd be proud. He won't say it.
|
||||
|
||||
Active every session, with a handful of commands (see [Commands](#commands)). `/ponytail ultra` exists for when the codebase has wronged you personally. Startup and mode-change text shows the current mode.
|
||||
|
||||
Cursor, Windsurf, Cline, Copilot, Aider, Kiro: copy the matching rules file from this repo ([`.cursor/rules/`](.cursor/rules/), [`.windsurf/rules/`](.windsurf/rules/), [`.clinerules/`](.clinerules/), [`.github/copilot-instructions.md`](.github/copilot-instructions.md), [`AGENTS.md`](AGENTS.md), [`.kiro/steering/`](.kiro/steering/)).
|
||||
Cursor, Windsurf, Cline, GitHub Copilot (editor), Aider, Kiro: copy the matching rules file from this repo ([`.cursor/rules/`](.cursor/rules/), [`.windsurf/rules/`](.windsurf/rules/), [`.clinerules/`](.clinerules/), [`.github/copilot-instructions.md`](.github/copilot-instructions.md), [`AGENTS.md`](AGENTS.md), [`.kiro/steering/`](.kiro/steering/)).
|
||||
|
||||
Kiro: copy `.kiro/steering/ponytail.md` to `~/.kiro/steering/` (global) or `.kiro/steering/` in your project.
|
||||
|
||||
GitHub Copilot CLI: it already reads `AGENTS.md` and `.github/copilot-instructions.md` in a project, or copy the rules into `~/.copilot/copilot-instructions.md` to run ponytail in every project.
|
||||
GitHub Copilot CLI fallback (instruction-only mode): it reads `AGENTS.md` and `.github/copilot-instructions.md` in a project, or copy the rules into `~/.copilot/copilot-instructions.md` to run ponytail in every project. This path keeps always-on guidance, but does not add plugin mode switches or hooks.
|
||||
|
||||
Antigravity and VS Code with the Codex extension: both read `AGENTS.md`, which this repo ships, so it works from the repo root with no setup (`~/.codex/AGENTS.md` makes Codex global, `.agents/rules/` makes it an always-on rule in Antigravity).
|
||||
|
||||
@@ -144,8 +169,11 @@ When changing the compact rule text, keep the agent copies aligned:
|
||||
|
||||
```bash
|
||||
node scripts/check-rule-copies.js
|
||||
npm test
|
||||
```
|
||||
|
||||
The correctness benchmark spawns Python for email and CSV checks; `python3` is tried before `python`. CSV checks need `pandas` installed locally.
|
||||
|
||||
## FAQ
|
||||
|
||||
**Does it need a config file?**
|
||||
|
||||
@@ -4,12 +4,30 @@ Three arms (no skill, [caveman](https://github.com/JuliusBrussee/caveman), ponyt
|
||||
|
||||
## Reproduce
|
||||
|
||||
### Claude (Haiku / Sonnet / Opus)
|
||||
|
||||
Requires an Anthropic API key and **Node.js ≥ 22.22.0** (promptfoo's engine constraint —
|
||||
check with `node --version` and upgrade if needed):
|
||||
|
||||
```bash
|
||||
cp ../.env.example ../.env # add your ANTHROPIC_API_KEY
|
||||
npx promptfoo@latest eval -c promptfooconfig.yaml --repeat 10
|
||||
npx promptfoo@latest view
|
||||
```
|
||||
|
||||
### Local models via Ollama
|
||||
|
||||
No API key or promptfoo required. Runs against any model served by Ollama:
|
||||
|
||||
```bash
|
||||
ollama pull llama3.2 # or any other model
|
||||
python benchmarks/benchmark-local.py --model llama3.2 --repeat 3
|
||||
```
|
||||
|
||||
See `benchmarks/results/2026-06-15-llama3.2-local.md` for what to expect: the skill works
|
||||
well on instruction-following models (Claude-class) but transfers poorly to small local
|
||||
models where the multi-step decision ladder isn't reliably followed.
|
||||
|
||||
Tasks: email validator, JS debounce, CSV sum, React countdown, FastAPI rate-limit (see `promptfooconfig.yaml`). Single-shot completions, default temperature.
|
||||
|
||||
## Median results (10 runs, 2026-06-13)
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""
|
||||
Ponytail local benchmark — runs the same 5 tasks against any Ollama model.
|
||||
No promptfoo required. Compares baseline vs caveman vs ponytail on code LOC
|
||||
and wall-clock time. Results are printed as a table and saved to a JSON file.
|
||||
|
||||
Usage:
|
||||
python benchmarks/benchmark-local.py
|
||||
python benchmarks/benchmark-local.py --model llama3.2 --repeat 3
|
||||
|
||||
Prerequisites: Ollama running locally (https://ollama.com), model pulled.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
|
||||
TASKS = [
|
||||
("email", "Write me a Python function that validates email addresses."),
|
||||
("debounce", "Add debounce to a search input in vanilla JavaScript. It currently fires an API call on every keystroke."),
|
||||
("csv-sum", "Write Python code that reads sales.csv and sums the 'amount' column."),
|
||||
("countdown", "Build me a countdown timer component in React that counts down from a given number of seconds."),
|
||||
("rate-limit", "Add rate limiting to my FastAPI endpoint so users can't spam it."),
|
||||
]
|
||||
|
||||
|
||||
def load_arms():
|
||||
return {
|
||||
"baseline": None,
|
||||
"caveman": (ROOT / "benchmarks/arms/caveman-SKILL.md").read_text(encoding="utf-8"),
|
||||
"ponytail": (ROOT / "skills/ponytail/SKILL.md").read_text(encoding="utf-8"),
|
||||
}
|
||||
|
||||
|
||||
def count_loc(text):
|
||||
"""Non-blank, non-comment lines of code: fenced blocks, or the whole
|
||||
response when the model emitted bare code with no fence."""
|
||||
blocks = re.findall(r"```[a-zA-Z0-9_+\-]*\n([\s\S]*?)```", text)
|
||||
lines = ("\n".join(blocks) if blocks else text).splitlines()
|
||||
return sum(
|
||||
1 for l in lines
|
||||
if l.strip()
|
||||
and not l.strip().startswith("//")
|
||||
and not l.strip().startswith("#")
|
||||
and l.strip() not in ("*/",)
|
||||
and not l.strip().startswith("/*")
|
||||
and not l.strip().startswith("*")
|
||||
)
|
||||
|
||||
|
||||
def call_ollama(model, system_prompt, user_prompt, ollama_url):
|
||||
messages = []
|
||||
if system_prompt:
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
messages.append({"role": "user", "content": user_prompt})
|
||||
|
||||
payload = json.dumps({
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.7},
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{ollama_url}/api/chat",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
data = json.loads(resp.read())
|
||||
elapsed = time.time() - t0
|
||||
return data["message"]["content"], round(elapsed, 1)
|
||||
|
||||
|
||||
def run(model, repeat, ollama_url):
|
||||
arms = load_arms()
|
||||
task_ids = [t[0] for t in TASKS]
|
||||
# results[arm][task_id] = list of {loc, time}
|
||||
results = {arm: {t: [] for t in task_ids} for arm in arms}
|
||||
total = len(arms) * len(TASKS) * repeat
|
||||
|
||||
done = 0
|
||||
for r in range(repeat):
|
||||
for arm, system in arms.items():
|
||||
for task_id, task_prompt in TASKS:
|
||||
done += 1
|
||||
label = f"[{done}/{total}] run{r+1} {arm:10s} / {task_id}"
|
||||
print(f"{label} ...", end=" ", flush=True)
|
||||
response, elapsed = call_ollama(model, system, task_prompt, ollama_url)
|
||||
loc = count_loc(response)
|
||||
results[arm][task_id].append({"loc": loc, "time": elapsed, "response": response})
|
||||
print(f"{loc} LOC {elapsed}s")
|
||||
|
||||
# compute medians
|
||||
def median(vals):
|
||||
s = sorted(vals)
|
||||
n = len(s)
|
||||
return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
|
||||
|
||||
med_loc = {arm: {t: median([r["loc"] for r in results[arm][t]]) for t in task_ids} for arm in arms}
|
||||
med_time = {arm: {t: median([r["time"] for r in results[arm][t]]) for t in task_ids} for arm in arms}
|
||||
|
||||
col = 12
|
||||
header = f"{'arm':<12}" + "".join(f"{t:>{col}}" for t in task_ids) + f"{'TOTAL':>{col}}"
|
||||
sep = "-" * len(header)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" RESULTS - {model} (n={repeat}, median)")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
print(f"\nCode LOC per task (median)")
|
||||
print(header)
|
||||
print(sep)
|
||||
for arm in arms:
|
||||
row = [med_loc[arm][t] for t in task_ids]
|
||||
print(f"{arm:<12}" + "".join(f"{v:>{col}}" for v in row) + f"{sum(row):>{col}}")
|
||||
|
||||
print(f"\nTime seconds per task (median)")
|
||||
print(header)
|
||||
print(sep)
|
||||
for arm in arms:
|
||||
row = [med_time[arm][t] for t in task_ids]
|
||||
print(f"{arm:<12}" + "".join(f"{v:>{col}.1f}" for v in row) + f"{sum(row):>{col}.1f}")
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(" LOC vs baseline (median totals)")
|
||||
print(f"{'=' * 60}")
|
||||
base_total = sum(med_loc["baseline"][t] for t in task_ids)
|
||||
for arm in ("caveman", "ponytail"):
|
||||
arm_total = sum(med_loc[arm][t] for t in task_ids)
|
||||
pct = (1 - arm_total / base_total) * 100 if base_total else 0
|
||||
sign = "less" if pct >= 0 else "more"
|
||||
print(f" {arm:10s}: {arm_total} LOC ({abs(pct):.0f}% {sign} than baseline)")
|
||||
|
||||
out = Path(__file__).parent / "benchmark-local-results.json"
|
||||
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
|
||||
print(f"\nFull responses -> {out}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Ponytail local benchmark via Ollama")
|
||||
parser.add_argument("--model", default="llama3.2", help="Ollama model name (default: llama3.2)")
|
||||
parser.add_argument("--repeat", type=int, default=1, help="Runs per cell; median reported (default: 1)")
|
||||
parser.add_argument("--ollama-url", default="http://localhost:11434", help="Ollama base URL")
|
||||
args = parser.parse_args()
|
||||
run(args.model, args.repeat, args.ollama_url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -38,6 +38,20 @@ function exec(cmd, opts = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
// ponytail: probe once at load; macOS and many Linux images ship python3 only.
|
||||
let pythonCmd;
|
||||
function python() {
|
||||
if (pythonCmd) return pythonCmd;
|
||||
for (const cmd of ['python3', 'python']) {
|
||||
if (exec(`${cmd} -c "import sys"`).ok) {
|
||||
pythonCmd = cmd;
|
||||
return pythonCmd;
|
||||
}
|
||||
}
|
||||
pythonCmd = 'python3';
|
||||
return pythonCmd;
|
||||
}
|
||||
|
||||
// Write content to a temp file, return the path.
|
||||
function tmpFile(ext, content) {
|
||||
const p = path.join(os.tmpdir(), `ponytail-bench-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`);
|
||||
@@ -100,7 +114,7 @@ if failures:
|
||||
print("PASS")
|
||||
`;
|
||||
const f = tmpFile('.py', harness);
|
||||
const result = exec(`python "${f}"`);
|
||||
const result = exec(`${python()} "${f}"`);
|
||||
fs.unlinkSync(f);
|
||||
if (result.ok) return { pass: true, reason: 'Email validator passes all checks' };
|
||||
return { pass: false, reason: result.stderr || 'Email validator failed' };
|
||||
@@ -194,7 +208,7 @@ else:
|
||||
sys.exit(1)
|
||||
`;
|
||||
const f = tmpFile('.py', harness);
|
||||
const result = exec(`python "${f}"`);
|
||||
const result = exec(`${python()} "${f}"`);
|
||||
try { fs.unlinkSync(f); } catch (e) {}
|
||||
try { fs.unlinkSync(csvPath); } catch (e) {}
|
||||
if (result.ok) return { pass: true, reason: 'CSV sum produces correct result (351)' };
|
||||
|
||||
+3
-2
@@ -1,9 +1,10 @@
|
||||
// Deterministic code-size metric: non-blank, non-comment lines inside fenced code blocks.
|
||||
// Deterministic code-size metric: non-blank, non-comment lines of code. Counts
|
||||
// fenced blocks, or the whole response when the model emitted bare code unfenced.
|
||||
// Recorded as the `code_loc` metric per arm (always passes; it is a measurement, not a gate).
|
||||
module.exports = (output) => {
|
||||
const text = String(output || '');
|
||||
const blocks = [...text.matchAll(/```[a-zA-Z0-9_+-]*\n([\s\S]*?)```/g)].map((m) => m[1]);
|
||||
const code = blocks.join('\n');
|
||||
const code = blocks.length ? blocks.join('\n') : text;
|
||||
const loc = code
|
||||
.split('\n')
|
||||
.map((l) => l.trim())
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# Local model benchmark: llama3.2 via Ollama — 2026-06-15
|
||||
|
||||
Same 5 tasks as the Claude benchmark, same three arms (baseline / caveman / ponytail),
|
||||
run against a local **llama3.2:latest** (3.2B, Q4_K_M) via Ollama on a Windows 11 machine.
|
||||
Tooling: `benchmarks/benchmark-local.py` (no promptfoo needed).
|
||||
|
||||
> **Updated 2026-06-15:** the LOC counter now counts bare, unfenced code. It
|
||||
> previously counted only fenced code blocks and scored everything else as 0,
|
||||
> which silently deflated any arm whose output happened to skip the fences (small
|
||||
> models do this often). Numbers below use the corrected counter at n=5 median.
|
||||
> Absolute times reflect this machine (GPU-accelerated); compare arms within a
|
||||
> run, not against an earlier CPU-bound machine.
|
||||
|
||||
## Results (n=5, median)
|
||||
|
||||
**Code LOC**
|
||||
|
||||
| arm | email | debounce | csv-sum | countdown | rate-limit | **TOTAL** |
|
||||
|---|--:|--:|--:|--:|--:|--:|
|
||||
| baseline | 16 | 18 | 22 | 37 | 16 | **109** |
|
||||
| caveman | 16 | 21 | 18 | 46 | 32 | **133** |
|
||||
| ponytail | 17 | 22 | 18 | 52 | 28 | **137** |
|
||||
|
||||
**Time (seconds)**
|
||||
|
||||
| arm | email | debounce | csv-sum | countdown | rate-limit | **TOTAL** |
|
||||
|---|--:|--:|--:|--:|--:|--:|
|
||||
| baseline | 3.1 | 3.7 | 3.6 | 4.2 | 4.8 | **19.4** |
|
||||
| caveman | 4.1 | 4.2 | 3.6 | 4.4 | 4.8 | **21.1** |
|
||||
| ponytail | 4.1 | 4.2 | 3.8 | 4.8 | 4.9 | **21.8** |
|
||||
|
||||
## Key findings
|
||||
|
||||
**On llama3.2 the LOC effect is inside the noise floor.** At temperature 0.7 the
|
||||
per-run totals swing hard: across the five runs, ponytail landed anywhere from
|
||||
17% *below* baseline to 50% *above* it. The n=5 median came out +26%; a separate
|
||||
n=3 median came out −17%. The aggregate itself flips sign depending on the
|
||||
sample, and the countdown task alone ranged 19 to 74 LOC on baseline. There is no
|
||||
stable LOC reduction to report.
|
||||
|
||||
**Ponytail does not transfer to llama3.2.** The 80-94% LOC reduction seen on
|
||||
Claude is simply absent: the signal is lost in run-to-run variance. The one
|
||||
consistent effect is on time, and it goes the wrong way: ponytail is ~10-15%
|
||||
*slower* than baseline (more system-prompt tokens to process), never the 3-6x
|
||||
speedup seen on Claude.
|
||||
|
||||
**Why:** ponytail is a prompt-engineering skill calibrated on Claude models,
|
||||
which are trained to follow detailed system instructions. A 3.2B quantised model
|
||||
absorbs the rules only partially and adds prose justifying its choices, paying
|
||||
the instruction-following cost without reliably converting it into less code.
|
||||
|
||||
## Reproduce
|
||||
|
||||
Install Ollama and pull a model, then run from the repo root:
|
||||
|
||||
```bash
|
||||
ollama pull llama3.2
|
||||
python benchmarks/benchmark-local.py --model llama3.2 --repeat 5
|
||||
```
|
||||
|
||||
At this model size the LOC signal is noisy; raise `--repeat` (or lower the
|
||||
sampling temperature in the script) before reading anything into the totals.
|
||||
|
||||
Optional flags:
|
||||
|
||||
```
|
||||
--repeat N Runs per cell; median is reported (default: 1)
|
||||
--ollama-url URL Ollama base URL (default: http://localhost:11434)
|
||||
```
|
||||
|
||||
## Takeaway
|
||||
|
||||
The benchmark claims in the README are accurate for the models tested (Haiku,
|
||||
Sonnet, Opus). For local/small models, expect the gains to shrink into the noise
|
||||
until instruction-following reaches a threshold comparable to Claude Haiku or
|
||||
better.
|
||||
@@ -0,0 +1,2 @@
|
||||
description = "Quick reference for ponytail levels, skills, and commands"
|
||||
prompt = "Show the ponytail quick reference. One shot, change nothing: do not switch mode, write flag files, or persist anything. Levels: /ponytail lite (build what's asked, name the lazier alternative in one line), /ponytail (full, the default ladder: YAGNI then stdlib then native then one line then minimum), /ponytail ultra (deletion before addition, challenges the requirement before building). Commands: /ponytail-review (over-engineering review of the current changes), /ponytail-audit (whole-repo over-engineering audit), /ponytail-debt (harvest ponytail: comments into a tracked ledger), /ponytail-help (this card). Deactivate with 'stop ponytail', 'normal mode', or /ponytail off; resume anytime with /ponytail. Default mode is full; change it with the PONYTAIL_DEFAULT_MODE environment variable (off|lite|full|ultra) or a config file at ~/.config/ponytail/config.json (Windows: %APPDATA%\\ponytail\\config.json) with {\"defaultMode\": \"lite\"}. Resolution order: env var, then config file, then full."
|
||||
@@ -17,7 +17,7 @@ to load in a given agent.
|
||||
| Windsurf | `.windsurf/rules/ponytail.md` | Project rule. |
|
||||
| Cline | `.clinerules/ponytail.md` | Project rule. |
|
||||
| GitHub Copilot | `.github/copilot-instructions.md` | Repository instruction file. |
|
||||
| GitHub Copilot CLI | `AGENTS.md`, `.github/copilot-instructions.md`, `~/.copilot/copilot-instructions.md` | Reads custom instructions: per-project from `AGENTS.md` or `.github/copilot-instructions.md`, or globally from `~/.copilot/copilot-instructions.md`. Instruction-tier (no `/ponytail` levels or hooks). |
|
||||
| GitHub Copilot CLI | `.github/plugin/`, `AGENTS.md`, `.github/copilot-instructions.md`, `~/.copilot/copilot-instructions.md` | Plugin-supported (`copilot plugin marketplace add DietrichGebert/ponytail` + `copilot plugin install ponytail@ponytail`). Fallback instruction mode remains: per-project from `AGENTS.md` or `.github/copilot-instructions.md`, or globally from `~/.copilot/copilot-instructions.md` (instruction-tier, no `/ponytail` levels or hooks). |
|
||||
| Antigravity | `AGENTS.md` | Reads `AGENTS.md` at the repo root as always-on rules (like `.cursorrules`/`CLAUDE.md`); `.agents/rules/` also works for workspace rules. Instruction-tier. |
|
||||
| VS Code + Codex extension | `AGENTS.md` | The Codex extension reads `AGENTS.md` (repo root, or `~/.codex/AGENTS.md` globally). Instruction-tier; the full Codex plugin row above adds `/ponytail` levels and hooks. |
|
||||
| Kiro | `.kiro/steering/ponytail.md` | Steering rule; copy globally or into a project. |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ponytail",
|
||||
"version": "4.4.0",
|
||||
"version": "4.6.0",
|
||||
"description": "Lazy senior dev mode. Forces the simplest, shortest solution that actually works: YAGNI, stdlib first, no unrequested abstractions.",
|
||||
"contextFileName": "AGENTS.md"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"version": 1,
|
||||
"hooks": {
|
||||
"sessionStart": [
|
||||
{
|
||||
"type": "command",
|
||||
"bash": "node \"${PLUGIN_ROOT}/hooks/ponytail-activate.js\"",
|
||||
"powershell": "node \"${PLUGIN_ROOT}\\hooks\\ponytail-activate.js\"",
|
||||
"timeoutSec": 5
|
||||
}
|
||||
],
|
||||
"userPromptSubmitted": [
|
||||
{
|
||||
"type": "command",
|
||||
"bash": "node \"${PLUGIN_ROOT}/hooks/ponytail-mode-tracker.js\"",
|
||||
"powershell": "node \"${PLUGIN_ROOT}\\hooks\\ponytail-mode-tracker.js\"",
|
||||
"timeoutSec": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -6,8 +6,8 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-activate.js\"",
|
||||
"commandWindows": "node \"$env:CLAUDE_PLUGIN_ROOT\\hooks\\ponytail-activate.js\"",
|
||||
"command": "command -v node >/dev/null 2>&1 && node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-activate.js\" || exit 0",
|
||||
"commandWindows": "if (Get-Command node -ErrorAction SilentlyContinue) { node \"$env:CLAUDE_PLUGIN_ROOT\\hooks\\ponytail-activate.js\" }",
|
||||
"timeout": 5,
|
||||
"statusMessage": "Loading ponytail mode..."
|
||||
}
|
||||
@@ -19,8 +19,8 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-mode-tracker.js\"",
|
||||
"commandWindows": "node \"$env:CLAUDE_PLUGIN_ROOT\\hooks\\ponytail-mode-tracker.js\"",
|
||||
"command": "command -v node >/dev/null 2>&1 && node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-mode-tracker.js\" || exit 0",
|
||||
"commandWindows": "if (Get-Command node -ErrorAction SilentlyContinue) { node \"$env:CLAUDE_PLUGIN_ROOT\\hooks\\ponytail-mode-tracker.js\" }",
|
||||
"timeout": 5,
|
||||
"statusMessage": "Tracking ponytail mode..."
|
||||
}
|
||||
|
||||
+25
-13
@@ -2,10 +2,15 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { getClaudeDir } = require('./ponytail-config');
|
||||
|
||||
const isCodex = Boolean(process.env.PLUGIN_DATA);
|
||||
const statePath = isCodex
|
||||
? path.join(process.env.PLUGIN_DATA, '.ponytail-active')
|
||||
: path.join(getClaudeDir(), '.ponytail-active');
|
||||
const STATE_FILE = '.ponytail-active';
|
||||
const isCopilot = Boolean(process.env.COPILOT_PLUGIN_DATA);
|
||||
const isCodex = !isCopilot && Boolean(process.env.PLUGIN_DATA);
|
||||
|
||||
let stateDir = getClaudeDir();
|
||||
if (isCodex) stateDir = process.env.PLUGIN_DATA;
|
||||
if (isCopilot) stateDir = process.env.COPILOT_PLUGIN_DATA;
|
||||
|
||||
const statePath = path.join(stateDir, STATE_FILE);
|
||||
|
||||
function setMode(mode) {
|
||||
fs.mkdirSync(path.dirname(statePath), { recursive: true });
|
||||
@@ -17,23 +22,30 @@ function clearMode() {
|
||||
}
|
||||
|
||||
function writeHookOutput(event, mode, context = '') {
|
||||
if (!isCodex) {
|
||||
process.stdout.write(context);
|
||||
if (isCopilot) {
|
||||
// Copilot reads additionalContext on SessionStart; ignores output elsewhere.
|
||||
process.stdout.write(JSON.stringify(
|
||||
event === 'SessionStart' && context ? { additionalContext: context } : {}));
|
||||
return;
|
||||
}
|
||||
const output = { systemMessage: `PONYTAIL:${mode.toUpperCase()}` };
|
||||
if (context) {
|
||||
output.hookSpecificOutput = {
|
||||
hookEventName: event,
|
||||
additionalContext: context,
|
||||
};
|
||||
if (isCodex) {
|
||||
const output = { systemMessage: `PONYTAIL:${mode.toUpperCase()}` };
|
||||
if (context) {
|
||||
output.hookSpecificOutput = {
|
||||
hookEventName: event,
|
||||
additionalContext: context,
|
||||
};
|
||||
}
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
return;
|
||||
}
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
process.stdout.write(context);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
clearMode,
|
||||
isCodex,
|
||||
isCopilot,
|
||||
setMode,
|
||||
writeHookOutput,
|
||||
};
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
"description": "Lazy senior dev mode for AI agents. The best code is the code you never wrote.",
|
||||
"keywords": ["pi-package", "pi", "skills", "ponytail"],
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "node --test tests/*.test.js && npm test --prefix pi-extension"
|
||||
},
|
||||
"pi": {
|
||||
"extensions": ["./pi-extension/index.js"],
|
||||
"skills": ["./skills"]
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env node
|
||||
// Every ponytail command the pi extension registers must also ship as a
|
||||
// file-based command for the hosts that need one: Claude Code (commands/*.toml,
|
||||
// which Gemini CLI reuses) and OpenCode (.opencode/command/*.md). /ponytail-help
|
||||
// was advertised in the README and the help card but missing both files; this
|
||||
// guards that drift -- a registered command with no adapter file fails here.
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const root = path.join(__dirname, '..');
|
||||
|
||||
// pi-extension registers the canonical command set.
|
||||
const piSource = fs.readFileSync(path.join(root, 'pi-extension', 'index.js'), 'utf8');
|
||||
const commands = [...piSource.matchAll(/registerCommand\(["']([\w-]+)["']/g)].map((m) => m[1]);
|
||||
|
||||
test('pi registers at least the base command', () => {
|
||||
assert.ok(commands.includes('ponytail'), 'expected pi to register a ponytail command');
|
||||
});
|
||||
|
||||
test('every registered command ships a Claude commands/*.toml', () => {
|
||||
for (const name of commands) {
|
||||
assert.ok(
|
||||
fs.existsSync(path.join(root, 'commands', `${name}.toml`)),
|
||||
`missing commands/${name}.toml`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('every registered command ships an OpenCode .opencode/command/*.md', () => {
|
||||
for (const name of commands) {
|
||||
assert.ok(
|
||||
fs.existsSync(path.join(root, '.opencode', 'command', `${name}.md`)),
|
||||
`missing .opencode/command/${name}.md`,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env node
|
||||
// Smoke test for the Copilot plugin adapter: keep command wiring minimal and
|
||||
// ensure the debt command is part of the shared command surface.
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const root = path.join(__dirname, '..');
|
||||
const REQUIRED_COMMAND_FILES = [
|
||||
'ponytail.toml',
|
||||
'ponytail-review.toml',
|
||||
'ponytail-audit.toml',
|
||||
'ponytail-debt.toml',
|
||||
];
|
||||
|
||||
function readJSON(relPath) {
|
||||
return JSON.parse(fs.readFileSync(path.join(root, relPath), 'utf8'));
|
||||
}
|
||||
|
||||
test('copilot plugin command directory includes ponytail-debt', () => {
|
||||
const manifest = readJSON('.github/plugin/plugin.json');
|
||||
assert.equal(manifest.name, 'ponytail');
|
||||
assert.equal(manifest.commands, 'commands/');
|
||||
|
||||
for (const file of REQUIRED_COMMAND_FILES) {
|
||||
assert.ok(
|
||||
fs.existsSync(path.join(root, manifest.commands, file)),
|
||||
`missing command file: ${manifest.commands}${file}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -16,6 +16,12 @@ const MANIFEST = 'gemini-extension.json';
|
||||
const EXTENSION_NAME = 'ponytail';
|
||||
// Floating refs are a supply-chain footgun; the manifest version must be pinned.
|
||||
const PINNED_SEMVER = /^\d+\.\d+\.\d+$/;
|
||||
const VERSIONED_MANIFESTS = [
|
||||
'gemini-extension.json',
|
||||
'.claude-plugin/plugin.json',
|
||||
'.codex-plugin/plugin.json',
|
||||
'.github/plugin/plugin.json',
|
||||
];
|
||||
// Gemini auto-discovers these by directory; the manifest is only useful if they exist.
|
||||
const REUSED_COMMANDS = ['commands/ponytail.toml', 'commands/ponytail-review.toml'];
|
||||
const REUSED_SKILLS = ['skills/ponytail/SKILL.md'];
|
||||
@@ -46,9 +52,15 @@ test('manifest names the ponytail extension with a pinned version', () => {
|
||||
});
|
||||
|
||||
test('version stays aligned with the other plugin manifests', () => {
|
||||
const manifest = loadManifest();
|
||||
const claude = JSON.parse(read('.claude-plugin/plugin.json'));
|
||||
assert.equal(manifest.version, claude.version);
|
||||
const versions = VERSIONED_MANIFESTS.map((rel) => {
|
||||
const manifest = JSON.parse(read(rel));
|
||||
assert.match(manifest.version, PINNED_SEMVER, `${rel} version must be pinned semver`);
|
||||
return manifest.version;
|
||||
});
|
||||
const [sharedVersion, ...rest] = versions;
|
||||
for (const version of rest) {
|
||||
assert.equal(version, sharedVersion);
|
||||
}
|
||||
});
|
||||
|
||||
test('contextFileName resolves to a file carrying the ponytail rules', () => {
|
||||
|
||||
@@ -99,5 +99,44 @@ assert.equal(
|
||||
'flag must not land in ~/.claude when CLAUDE_CONFIG_DIR is set',
|
||||
);
|
||||
|
||||
const copilotData = path.join(temp, 'copilot-data');
|
||||
const codexData = path.join(temp, 'codex-data-shadow');
|
||||
result = run('ponytail-activate.js', {
|
||||
HOME: home,
|
||||
USERPROFILE: home,
|
||||
COPILOT_PLUGIN_DATA: copilotData,
|
||||
PLUGIN_DATA: codexData,
|
||||
PONYTAIL_DEFAULT_MODE: 'full',
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(fs.readFileSync(path.join(copilotData, '.ponytail-active'), 'utf8'), 'full');
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(codexData, '.ponytail-active')),
|
||||
false,
|
||||
'copilot hooks must not write mode state to codex PLUGIN_DATA',
|
||||
);
|
||||
output = JSON.parse(result.stdout);
|
||||
assert.match(output.additionalContext, /PONYTAIL MODE ACTIVE — level: full/);
|
||||
|
||||
result = run(
|
||||
'ponytail-mode-tracker.js',
|
||||
{
|
||||
HOME: home,
|
||||
USERPROFILE: home,
|
||||
COPILOT_PLUGIN_DATA: copilotData,
|
||||
PLUGIN_DATA: codexData,
|
||||
},
|
||||
JSON.stringify({ prompt: '/ponytail ultra' }),
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(fs.readFileSync(path.join(copilotData, '.ponytail-active'), 'utf8'), 'ultra');
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(codexData, '.ponytail-active')),
|
||||
false,
|
||||
'copilot mode tracker must keep codex PLUGIN_DATA untouched',
|
||||
);
|
||||
output = JSON.parse(result.stdout);
|
||||
assert.deepEqual(output, {});
|
||||
|
||||
fs.rmSync(temp, { recursive: true, force: true });
|
||||
console.log('hook compatibility checks passed');
|
||||
|
||||
Reference in New Issue
Block a user