DietrichGebertandClaude Opus 4.8
2e6a93765a
Fixes the local benchmark LOC counter (counted only fenced code, scored bare output 0), makes summary output ASCII-safe (a Unicode arrow crashed the script on Windows cp1252), gitignores generated artifacts, and refreshes the llama3.2 writeup with n=5 data showing the LOC effect is within the noise floor. Follow-up to #63. Verified live. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
14 lines
738 B
JavaScript
14 lines
738 B
JavaScript
// Deterministic code-size metric: non-blank, non-comment lines of code. Counts
|
|
// fenced blocks, or the whole response when the model emitted bare code unfenced.
|
|
// Recorded as the `code_loc` metric per arm (always passes; it is a measurement, not a gate).
|
|
module.exports = (output) => {
|
|
const text = String(output || '');
|
|
const blocks = [...text.matchAll(/```[a-zA-Z0-9_+-]*\n([\s\S]*?)```/g)].map((m) => m[1]);
|
|
const code = blocks.length ? blocks.join('\n') : text;
|
|
const loc = code
|
|
.split('\n')
|
|
.map((l) => l.trim())
|
|
.filter((l) => l && !l.startsWith('//') && !l.startsWith('#') && l !== '*/' && !l.startsWith('/*') && !l.startsWith('*')).length;
|
|
return { pass: true, score: loc, reason: loc + ' code LOC' };
|
|
};
|