* feat: refine ruleset from a full-project field review
A reviewer ran ponytail across a 9-phase rewrite (protocol, PC app, simulator,
RPi daemon, ESP32 firmware) and flagged three gaps. All three land in SKILL.md
and propagate to AGENTS.md + the rule copies:
- Promote the one-runnable-check rule to a headline ("Lazy code without its
check is unfinished"), enforced as a check-rule-copies invariant.
- Hardware carve-out in "When NOT to be lazy": a real device is never the spec
ideal (clock drift, sensor offset), leave the calibration knob.
- Clarify the Output rule: explanation the user explicitly asked for is not
debt, only unrequested prose is.
Fallback instructions kept in sync. Rule-copy check + tests green.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* test: add a behavior gate proving the refinements actually fire
The refinements were verified as injected text, but injected != behavioral.
This adds a behavior eval that probes each refined rule on a task that should
trigger it:
- hardware -> does the output leave a calibration knob?
- explanation -> when a write-up is explicitly requested, is it given in full?
- onecheck -> is a runnable check left behind?
benchmarks/behavior.yaml runs the probes (baseline vs ponytail arm); the
grader benchmarks/behavior.js is proven by tests/behavior.test.js (8 cases,
RED/GREEN, no API key, runs in CI). Live-confirmed: the model under the
current ruleset passes all three gates, graded by the same grader.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
67 lines
2.4 KiB
JavaScript
67 lines
2.4 KiB
JavaScript
#!/usr/bin/env node
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const root = path.join(__dirname, '..');
|
|
|
|
function read(relPath) {
|
|
return fs.readFileSync(path.join(root, relPath), 'utf8').replace(/\r\n/g, '\n').trim();
|
|
}
|
|
|
|
function stripFrontmatter(text) {
|
|
return text.replace(/^---\n[\s\S]*?\n---\n*/, '').trim();
|
|
}
|
|
|
|
const agents = read('AGENTS.md');
|
|
const canonical = agents.replace(/\n\n\(Yes, this file also applies[\s\S]*?\)$/, '').trim();
|
|
|
|
// Compact copies: same body as AGENTS.md, host-specific frontmatter stripped.
|
|
const copies = [
|
|
['.cursor/rules/ponytail.mdc', stripFrontmatter],
|
|
['.windsurf/rules/ponytail.md', text => text.trim()],
|
|
['.clinerules/ponytail.md', text => text.trim()],
|
|
['.github/copilot-instructions.md', text => text.trim()],
|
|
['.kiro/steering/ponytail.md', stripFrontmatter],
|
|
];
|
|
|
|
let failed = false;
|
|
|
|
for (const [relPath, normalize] of copies) {
|
|
const actual = normalize(read(relPath));
|
|
if (actual !== canonical) {
|
|
console.error(`${relPath} drifted from AGENTS.md`);
|
|
failed = true;
|
|
}
|
|
}
|
|
|
|
// SKILL.md is the runtime source of truth and is longer than the compact body,
|
|
// so it cannot be byte-compared. ponytail: canary, not full equality. Assert the
|
|
// load-bearing rules survive verbatim in both the source and AGENTS.md. Changing
|
|
// a rule's wording trips this, which is the reminder to propagate it everywhere.
|
|
// Upgrade path: generate the copies from SKILL.md if this ever misses a real drift.
|
|
const INVARIANTS = [
|
|
'naive heuristic', // ceiling-comment rule
|
|
'ONE runnable check', // test reflex
|
|
'flimsier algorithm', // robust-variant rule
|
|
'input validation at trust boundaries', // the "not lazy about" clause
|
|
'Lazy code without its check is unfinished', // one-check promoted to headline
|
|
];
|
|
|
|
const skill = read('skills/ponytail/SKILL.md');
|
|
const sources = [['skills/ponytail/SKILL.md', skill], ['AGENTS.md', agents]];
|
|
for (const phrase of INVARIANTS) {
|
|
for (const [label, text] of sources) {
|
|
if (!text.includes(phrase)) {
|
|
console.error(`${label} is missing rule invariant: "${phrase}"`);
|
|
failed = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (failed) {
|
|
console.error('Update the copied rule text, AGENTS.md, or SKILL.md so the shared rules match.');
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`Rule copies match AGENTS.md; ${INVARIANTS.length} rule invariants present in SKILL.md and AGENTS.md.`);
|