Files
ponytail/benchmarks/loc.js
T
Lakshya Sharma 8d154e6c2a fix(benchmarks): strip block comments before counting LOC (#232)
loc.js filtered comments line by line, so /* ... */ block comments whose
continuation lines are not *-aligned had their inner lines counted as code.
A plain indented block comment scored higher than the same code written
JSDoc-style. Strip /* ... */ before the line count so both are equal, and
add loc.test.js to lock the behavior.

Fixes #231
2026-06-26 03:02:55 +02:00

16 lines
932 B
JavaScript

// Deterministic code-size metric: non-blank, non-comment lines of code. Counts
// fenced blocks, or the whole response when the model emitted bare code unfenced.
// Recorded as the `code_loc` metric per arm (always passes; it is a measurement, not a gate).
module.exports = (output) => {
const text = String(output || '');
const blocks = [...text.matchAll(/```[a-zA-Z0-9_+-]*\n([\s\S]*?)```/g)].map((m) => m[1]);
// Drop /* ... */ block comments before counting; the line filter below only
// caught `*`-aligned JSDoc, so plain block comments were miscounted as code.
const code = (blocks.length ? blocks.join('\n') : text).replace(/\/\*[\s\S]*?\*\//g, '');
const loc = code
.split('\n')
.map((l) => l.trim())
.filter((l) => l && !l.startsWith('//') && !l.startsWith('#') && l !== '*/' && !l.startsWith('/*') && !l.startsWith('*')).length;
return { pass: true, score: loc, reason: loc + ' code LOC' };
};