prototype/scripts/evaluate.mjs

import {readFile,writeFile} from 'node:fs/promises';
import {runTriage} from '../src/workflow.mjs';
import {embedTexts,issueText,MODEL} from '../src/model.mjs';
import {cosine,THRESHOLDS,LABELS,keywordBaseline,digest} from '../src/core.mjs';
globalThis.fetch=async()=>{throw new Error('Network disabled for the entire evaluation.');};
const fixture=JSON.parse(await readFile(new URL('../data/evaluation.json',import.meta.url),'utf8'));
const reference=JSON.parse(await readFile(new URL('../data/reference.json',import.meta.url),'utf8'));
const classes=[...LABELS,'unsure'];
const confusion=Object.fromEntries(classes.map(label=>[label,Object.fromEntries(classes.map(other=>[other,0]))]));
const baselineConfusion=structuredClone(confusion);
const first=await runTriage({issues:fixture.classification});
const warm=await runTriage({issues:fixture.classification});
const rows=first.items.map((item,index)=>{
  const expected=fixture.classification[index].expected,predicted=item.needsReview?'unsure':item.suggestedLabel,baseline=keywordBaseline(item);
  confusion[expected][predicted]++;baselineConfusion[expected][baseline]++;
  return {id:item.id,expected,predicted,rawLabel:item.suggestedLabel,similarity:item.similarity,margin:item.margin,baseline,correct:expected===predicted,needsReview:item.needsReview};
});
const pairs=[];
for(const pair of fixture.pairs){
  const [a,b]=await embedTexts([issueText(pair.a),issueText(pair.b)]),similarity=cosine(a,b);
  pairs.push({a:pair.a.id,b:pair.b.id,expected:pair.duplicate,predicted:similarity>=THRESHOLDS.duplicateSimilarity,similarity});
}
const truePositive=pairs.filter(x=>x.expected&&x.predicted).length,falsePositive=pairs.filter(x=>!x.expected&&x.predicted).length,falseNegative=pairs.filter(x=>x.expected&&!x.predicted).length;
const answered=rows.filter(x=>!x.needsReview),unknown=rows.filter(x=>x.expected==='unsure');
const stable=first.items.every((item,i)=>item.suggestedLabel===warm.items[i].suggestedLabel&&Math.abs(item.similarity-warm.items[i].similarity)<1e-7&&item.needsReview===warm.items[i].needsReview);
const output={available:true,createdAt:new Date().toISOString(),dataset:'fixed developer-authored synthetic holdout; not a real-world performance estimate',model:MODEL,thresholds:THRESHOLDS,referenceHash:digest(reference),fixtureHash:digest(fixture),classification:{total:rows.length,correct:rows.filter(x=>x.correct).length,accuracy:rows.filter(x=>x.correct).length/rows.length,baselineAccuracy:rows.filter(x=>x.expected===x.baseline).length/rows.length,abstained:rows.filter(x=>x.needsReview).length,coverage:answered.length/rows.length,selectiveAccuracy:answered.length?answered.filter(x=>x.correct).length/answered.length:null,unknownAbstention:unknown.filter(x=>x.needsReview).length,unknownTotal:unknown.length,confusion,baselineConfusion,rows},duplicates:{total:pairs.length,truePositive,falsePositive,falseNegative,precision:truePositive+falsePositive?truePositive/(truePositive+falsePositive):null,recall:truePositive+falseNegative?truePositive/(truePositive+falseNegative):null,pairs},runtime:{coldMs:first.timings.totalMs,warmMs:warm.timings.totalMs,rssMiB:Math.round(process.memoryUsage().rss/1024/1024),networkDisabled:true,deterministicAt1e7:stable},workflowExecution:first.execution,limitations:['Small synthetic fixture; no claim of real-world accuracy or human time saved.','Similarity is not a calibrated probability. All proposed labels need user review before export.','Long text is truncated at 256 wordpieces. English only; specialized issue vocabulary can fail.','Near-duplicate scores are suggestions, never automatic closure or merge.','Network-disabled proof blocks JavaScript fetch during inference; this is not an OS network sandbox.']};
await writeFile(new URL('../evidence/evaluation.json',import.meta.url),JSON.stringify(output,null,2)+'\n');
console.log(JSON.stringify({classification:{accuracy:output.classification.accuracy,baseline:output.classification.baselineAccuracy,coverage:output.classification.coverage},duplicates:output.duplicates,runtime:output.runtime},null,2));