Exact reviewed source / 15 of 64
prototype/evidence/evaluation.json
SHA-256 of original bytes
64fd2a52819e67ed22ba3c46fc3627e386b5b98f3c6aea9be95aacb6ececdb0c
{
"available": true,
"createdAt": "2026-10-19T22:34:23.145Z",
"dataset": "fixed developer-authored synthetic holdout; not a real-world performance estimate",
"model": {
"name": "Xenova/all-MiniLM-L6-v2",
"revision": "751bff37182d3f1213fa05d7196b954e230abad9",
"dtype": "q8",
"dimensions": 384,
"maxTokens": 256,
"license": "Apache-2.0",
"offline": true
},
"thresholds": {
"minimumSimilarity": 0.4,
"minimumMargin": 0.04,
"duplicateSimilarity": 0.83
},
"referenceHash": "44735dd6e345311b8129b9ef7ca96a6b5bce50b6ce497f06b8350fb7857b59e2",
"fixtureHash": "9ef7770c613dc08d0f6b8a8bd1226888d8a04b1ba53bea7b6ba8ed3bc0ac34bb",
"classification": {
"total": 20,
"correct": 9,
"accuracy": 0.45,
"baselineAccuracy": 0.9,
"abstained": 15,
"coverage": 0.25,
"selectiveAccuracy": 1,
"unknownAbstention": 4,
"unknownTotal": 4,
"confusion": {
"bug": {
"bug": 1,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 3
},
"feature": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 4
},
"documentation": {
"bug": 0,
"feature": 0,
"documentation": 3,
"question": 0,
"unsure": 1
},
"question": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 1,
"unsure": 3
},
"unsure": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 4
}
},
"baselineConfusion": {
"bug": {
"bug": 2,
"feature": 0,
"documentation": 0,
"question": 1,
"unsure": 1
},
"feature": {
"bug": 0,
"feature": 4,
"documentation": 0,
"question": 0,
"unsure": 0
},
"documentation": {
"bug": 0,
"feature": 0,
"documentation": 4,
"question": 0,
"unsure": 0
},
"question": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 4,
"unsure": 0
},
"unsure": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 4
}
},
"rows": [
{
"id": 201,
"expected": "bug",
"predicted": "unsure",
"rawLabel": "bug",
"similarity": 0.36021442782610946,
"margin": 0.20997445121461178,
"baseline": "bug",
"correct": false,
"needsReview": true
},
{
"id": 202,
"expected": "bug",
"predicted": "unsure",
"rawLabel": "bug",
"similarity": 0.28360056386632004,
"margin": 0.04581243080132563,
"baseline": "question",
"correct": false,
"needsReview": true
},
{
"id": 203,
"expected": "bug",
"predicted": "bug",
"rawLabel": "bug",
"similarity": 0.4921915358842254,
"margin": 0.28736290889695654,
"baseline": "unsure",
"correct": true,
"needsReview": false
},
{
"id": 204,
"expected": "bug",
"predicted": "unsure",
"rawLabel": "bug",
"similarity": 0.3061974964643349,
"margin": 0.13370857152333737,
"baseline": "bug",
"correct": false,
"needsReview": true
},
{
"id": 205,
"expected": "feature",
"predicted": "unsure",
"rawLabel": "feature",
"similarity": 0.25534207788299024,
"margin": 0.11855528133138724,
"baseline": "feature",
"correct": false,
"needsReview": true
},
{
"id": 206,
"expected": "feature",
"predicted": "unsure",
"rawLabel": "feature",
"similarity": 0.2333831080357983,
"margin": 0.061302636002256966,
"baseline": "feature",
"correct": false,
"needsReview": true
},
{
"id": 207,
"expected": "feature",
"predicted": "unsure",
"rawLabel": "question",
"similarity": 0.32094710754405315,
"margin": 0.04595118012955329,
"baseline": "feature",
"correct": false,
"needsReview": true
},
{
"id": 208,
"expected": "feature",
"predicted": "unsure",
"rawLabel": "feature",
"similarity": 0.34683807154249857,
"margin": 0.026751889736613577,
"baseline": "feature",
"correct": false,
"needsReview": true
},
{
"id": 209,
"expected": "documentation",
"predicted": "documentation",
"rawLabel": "documentation",
"similarity": 0.43820394426791076,
"margin": 0.1908172192397018,
"baseline": "documentation",
"correct": true,
"needsReview": false
},
{
"id": 210,
"expected": "documentation",
"predicted": "unsure",
"rawLabel": "documentation",
"similarity": 0.22723501637406013,
"margin": 0.08528916210155049,
"baseline": "documentation",
"correct": false,
"needsReview": true
},
{
"id": 211,
"expected": "documentation",
"predicted": "documentation",
"rawLabel": "documentation",
"similarity": 0.5116850581090184,
"margin": 0.29582712039450026,
"baseline": "documentation",
"correct": true,
"needsReview": false
},
{
"id": 212,
"expected": "documentation",
"predicted": "documentation",
"rawLabel": "documentation",
"similarity": 0.6379941982090976,
"margin": 0.3167043457419379,
"baseline": "documentation",
"correct": true,
"needsReview": false
},
{
"id": 213,
"expected": "question",
"predicted": "question",
"rawLabel": "question",
"similarity": 0.5040743875578764,
"margin": 0.3024009837060428,
"baseline": "question",
"correct": true,
"needsReview": false
},
{
"id": 214,
"expected": "question",
"predicted": "unsure",
"rawLabel": "question",
"similarity": 0.3601348794195268,
"margin": 0.22956607392002149,
"baseline": "question",
"correct": false,
"needsReview": true
},
{
"id": 215,
"expected": "question",
"predicted": "unsure",
"rawLabel": "question",
"similarity": 0.29099817371639164,
"margin": 0.07314081092102523,
"baseline": "question",
"correct": false,
"needsReview": true
},
{
"id": 216,
"expected": "question",
"predicted": "unsure",
"rawLabel": "documentation",
"similarity": 0.3912484497689992,
"margin": 0.052806230267178345,
"baseline": "question",
"correct": false,
"needsReview": true
},
{
"id": 217,
"expected": "unsure",
"predicted": "unsure",
"rawLabel": "feature",
"similarity": 0.1529623016373922,
"margin": 0.024112245018460904,
"baseline": "unsure",
"correct": true,
"needsReview": true
},
{
"id": 218,
"expected": "unsure",
"predicted": "unsure",
"rawLabel": "bug",
"similarity": 0.07631912079877232,
"margin": 0.024084821159938458,
"baseline": "unsure",
"correct": true,
"needsReview": true
},
{
"id": 219,
"expected": "unsure",
"predicted": "unsure",
"rawLabel": "bug",
"similarity": 0.16345005442474753,
"margin": 0.09576364003466203,
"baseline": "unsure",
"correct": true,
"needsReview": true
},
{
"id": 220,
"expected": "unsure",
"predicted": "unsure",
"rawLabel": "feature",
"similarity": 0.0956703883846812,
"margin": 0.029711023467636155,
"baseline": "unsure",
"correct": true,
"needsReview": true
}
]
},
"duplicates": {
"total": 6,
"truePositive": 2,
"falsePositive": 0,
"falseNegative": 1,
"precision": 1,
"recall": 0.6666666666666666,
"pairs": [
{
"a": 301,
"b": 302,
"expected": true,
"predicted": true,
"similarity": 0.967697647000363
},
{
"a": 303,
"b": 304,
"expected": true,
"predicted": true,
"similarity": 0.8528556119954988
},
{
"a": 305,
"b": 306,
"expected": true,
"predicted": false,
"similarity": 0.7824381158444897
},
{
"a": 307,
"b": 308,
"expected": false,
"predicted": false,
"similarity": 0.6160265627326454
},
{
"a": 309,
"b": 310,
"expected": false,
"predicted": false,
"similarity": 0.7018611566748348
},
{
"a": 311,
"b": 312,
"expected": false,
"predicted": false,
"similarity": 0.4825772775360722
}
]
},
"runtime": {
"coldMs": 215,
"warmMs": 46,
"rssMiB": 246,
"networkDisabled": true,
"deterministicAt1e7": true
},
"workflowExecution": {
"status": "success",
"steps": [
{
"id": "input"
},
{
"id": "validate-export",
"status": "success"
},
{
"id": "embed-locally",
"status": "success"
},
{
"id": "compare-and-abstain",
"status": "success"
},
{
"id": "build-review-queue",
"status": "success"
}
]
},
"limitations": [
"Small synthetic fixture; no claim of real-world accuracy or human time saved.",
"Similarity is not a calibrated probability. All proposed labels need user review before export.",
"Long text is truncated at 256 wordpieces. English only; specialized issue vocabulary can fail.",
"Near-duplicate scores are suggestions, never automatic closure or merge.",
"Network-disabled proof blocks JavaScript fetch during inference; this is not an OS network sandbox."
]
}