review/score_review.py

"""Compute exact reported metrics from preserved, genuine prototype predictions."""
from pathlib import Path
from collections import Counter
import hashlib,json
ROOT=Path(__file__).resolve().parent
load=lambda name:json.loads((ROOT/name).read_text())
manifest=load('fixture_manifest.json')
assert all(hashlib.sha256((ROOT/f).read_bytes()).hexdigest()==s for f,s in manifest['files'].items()), 'Frozen fixture changed'
fixtures=load('heldout_classification.json')['issues']
dups=load('heldout_duplicates.json')['issues']
raw=load('raw_predictions.json')
labels=['bug','feature','documentation','question','unsure','rejected']
items={item['id']:item for item in raw['classification']['items']}
keywords=load('baseline.json')['keywords']

def baseline(issue):
 text=(issue['title']+' '+issue['body']).lower()
 scores=sorted([(sum(w in text for w in words),label) for label,words in keywords.items()],reverse=True)
 return scores[0][1] if scores[0][0]>0 and scores[0][0]>scores[1][0] else 'unsure'

def metrics(predictions):
 confusion={truth:{pred:0 for pred in labels} for truth in labels}
 for issue,pred in zip(fixtures,predictions):confusion[issue['expected_label']][pred]+=1
 n=len(fixtures);correct=sum(confusion[l][l] for l in labels)
 answered=[i for i,p in enumerate(predictions) if p not in ['unsure','rejected']]
 per={}
 for label in labels[:-1]:
  tp=confusion[label][label];fp=sum(confusion[x][label] for x in labels if x!=label);fn=sum(confusion[label][x] for x in labels if x!=label)
  precision=tp/(tp+fp) if tp+fp else 0;recall=tp/(tp+fn) if tp+fn else 0
  per[label]={'support':sum(confusion[label].values()),'precision':precision,'recall':recall,'f1':2*precision*recall/(precision+recall) if precision+recall else 0}
 return {'count':n,'correct':correct,'accuracy':correct/n,'confusionMatrix':confusion,'perLabel':per,'macroF1':sum(v['f1'] for v in per.values())/5,'abstentionCount':predictions.count('unsure'),'abstentionRate':predictions.count('unsure')/n,'rejectedCount':predictions.count('rejected'),'coverage':len(answered)/n,'accuracyAmongAnswered':sum(predictions[i]==fixtures[i]['expected_label'] for i in answered)/len(answered) if answered else None}
pred=[('unsure' if items[i['number']]['needsReview'] else items[i['number']]['suggestedLabel']) if i['number'] in items else 'rejected' for i in fixtures]
rawPred=[items[i['number']]['suggestedLabel'] if i['number'] in items else 'rejected' for i in fixtures]
basePred=[baseline(i) for i in fixtures]
classFailures=[{'fixtureId':i['id'],'title':i['title'],'expected':i['expected_label'],'observed':p,'rawSuggestedLabel':items.get(i['number'],{}).get('suggestedLabel'),'similarity':items.get(i['number'],{}).get('similarity'),'margin':items.get(i['number'],{}).get('margin')} for i,p in zip(fixtures,pred) if i['expected_label']!=p]
byId={i['number']:i for i in dups}
truePairs={tuple(sorted((i['number'],j['number']))) for i in dups for j in dups if i['number']<j['number'] and i['duplicate_group'] and i['duplicate_group']==j['duplicate_group']}
predPairs=set();errors=[];retrieval=[]
for item in raw['duplicates']['items']:
 identity=item['id'];cand=item.get('duplicateOf');nearest=item.get('nearestCandidateId')
 if cand is not None:
  if cand==identity or cand not in byId:errors.append({'id':identity,'duplicateOf':cand,'error':'invalid or self candidate'})
  else:predPairs.add(tuple(sorted((identity,cand))))
 if byId[identity]['duplicate_group']:
  matched=nearest in byId and nearest!=identity and byId[nearest]['duplicate_group']==byId[identity]['duplicate_group']
  retrieval.append({'id':identity,'nearestCandidateId':nearest,'similarity':item.get('nearestCandidateSimilarity'),'correct':matched})
tp=len(predPairs & truePairs);fp=predPairs-truePairs;missed=truePairs-predPairs
precision=tp/len(predPairs) if predPairs else None;recall=tp/len(truePairs)
dupMetrics={'items':len(dups),'trueUniquePairs':len(truePairs),'predictedUniquePairs':len(predPairs),'truePositivePairs':tp,'precision':precision,'recall':recall,'f1':2*precision*recall/(precision+recall) if precision and recall else 0,'falsePositivePairs':[list(p) for p in sorted(fp)],'missedPairs':[list(p) for p in sorted(missed)],'top1PositiveAnchors':len(retrieval),'top1Correct':sum(i['correct'] for i in retrieval),'top1Recall':sum(i['correct'] for i in retrieval)/len(retrieval),'top1Details':retrieval,'invalidCandidates':errors,'threshold':raw['thresholds']['duplicateSimilarity'],'interpretation':'Threshold suggestions point to an earlier item; nearest-candidate retrieval is across all other items. Scores are cosine similarity, not probabilities or proof.'}
repeated={i['id']:i for i in raw['warmRepeat']['items']}
repeatStable=all(all(item.get(key)==repeated[item['id']].get(key) for key in ['suggestedLabel','similarity','margin','needsReview','duplicateOf','nearestCandidateId']) for item in items.values())
refs=json.loads((ROOT.parent/'prototype/data/reference.json').read_text())
norm=lambda i: ' '.join((i['title']+' '+i['body']).lower().split())
referenceExactOverlap=[i['id'] for i in fixtures if norm(i) in {norm(r) for r in refs}]
result={'synthetic':True,'claims':'Small fixed synthetic diagnostic only; no human time savings measured and no real-repository generalization established.','fixtureManifestVerified':True,'referenceExactTextOverlap':referenceExactOverlap,'model':raw['model'],'thresholds':raw['thresholds'],'classification':metrics(pred),'rawTopLabelClassification':metrics(rawPred),'keywordBaseline':metrics(basePred),'classificationFailures':classFailures,'rejectedInputs':raw['rejected'],'duplicates':dupMetrics,'timingMs':{k:raw[k] for k in ['modelLoadMs','referenceEmbeddingMs','classificationWallMs','duplicateWallMs','warmRepeatWallMs']},'processPeakRssKiB':raw['processPeakRssKiB'],'nodeVersion':raw['nodeVersion'],'realVectorEvidence':raw['vectorEvidence'],'warmRepeatPredictionsIdentical':repeatStable,'sourceFreeze':load('source_freeze.json')}
(ROOT/'metrics.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps({k:result[k] for k in ['classification','keywordBaseline','duplicates','timingMs','processPeakRssKiB','warmRepeatPredictionsIdentical']},indent=2))