Exact reviewed source / 54 of 64
review/metrics.json
SHA-256 of original bytes
cf771971e57fd9dc9922802e08d9ee35dfdcad2023f3efa47f796d4510d5623e
{
"synthetic": true,
"claims": "Small fixed synthetic diagnostic only; no human time savings measured and no real-repository generalization established.",
"fixtureManifestVerified": true,
"referenceExactTextOverlap": [],
"model": {
"name": "Xenova/all-MiniLM-L6-v2",
"revision": "751bff37182d3f1213fa05d7196b954e230abad9",
"dtype": "q8",
"dimensions": 384,
"maxTokens": 256,
"license": "Apache-2.0",
"offline": true
},
"thresholds": {
"minimumSimilarity": 0.4,
"minimumMargin": 0.04,
"duplicateSimilarity": 0.83
},
"classification": {
"count": 40,
"correct": 15,
"accuracy": 0.375,
"confusionMatrix": {
"bug": {
"bug": 1,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 7,
"rejected": 0
},
"feature": {
"bug": 0,
"feature": 1,
"documentation": 0,
"question": 0,
"unsure": 7,
"rejected": 0
},
"documentation": {
"bug": 0,
"feature": 0,
"documentation": 4,
"question": 0,
"unsure": 4,
"rejected": 0
},
"question": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 2,
"unsure": 6,
"rejected": 0
},
"unsure": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 7,
"rejected": 1
},
"rejected": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 0,
"rejected": 0
}
},
"perLabel": {
"bug": {
"support": 8,
"precision": 1.0,
"recall": 0.125,
"f1": 0.2222222222222222
},
"feature": {
"support": 8,
"precision": 1.0,
"recall": 0.125,
"f1": 0.2222222222222222
},
"documentation": {
"support": 8,
"precision": 1.0,
"recall": 0.5,
"f1": 0.6666666666666666
},
"question": {
"support": 8,
"precision": 1.0,
"recall": 0.25,
"f1": 0.4
},
"unsure": {
"support": 8,
"precision": 0.22580645161290322,
"recall": 0.875,
"f1": 0.358974358974359
}
},
"macroF1": 0.37401709401709404,
"abstentionCount": 31,
"abstentionRate": 0.775,
"rejectedCount": 1,
"coverage": 0.2,
"accuracyAmongAnswered": 1.0
},
"rawTopLabelClassification": {
"count": 40,
"correct": 26,
"accuracy": 0.65,
"confusionMatrix": {
"bug": {
"bug": 6,
"feature": 0,
"documentation": 2,
"question": 0,
"unsure": 0,
"rejected": 0
},
"feature": {
"bug": 1,
"feature": 5,
"documentation": 1,
"question": 1,
"unsure": 0,
"rejected": 0
},
"documentation": {
"bug": 0,
"feature": 0,
"documentation": 8,
"question": 0,
"unsure": 0,
"rejected": 0
},
"question": {
"bug": 0,
"feature": 0,
"documentation": 1,
"question": 7,
"unsure": 0,
"rejected": 0
},
"unsure": {
"bug": 3,
"feature": 3,
"documentation": 1,
"question": 0,
"unsure": 0,
"rejected": 1
},
"rejected": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 0,
"rejected": 0
}
},
"perLabel": {
"bug": {
"support": 8,
"precision": 0.6,
"recall": 0.75,
"f1": 0.6666666666666665
},
"feature": {
"support": 8,
"precision": 0.625,
"recall": 0.625,
"f1": 0.625
},
"documentation": {
"support": 8,
"precision": 0.6153846153846154,
"recall": 1.0,
"f1": 0.761904761904762
},
"question": {
"support": 8,
"precision": 0.875,
"recall": 0.875,
"f1": 0.875
},
"unsure": {
"support": 8,
"precision": 0,
"recall": 0.0,
"f1": 0
}
},
"macroF1": 0.5857142857142856,
"abstentionCount": 0,
"abstentionRate": 0.0,
"rejectedCount": 1,
"coverage": 0.975,
"accuracyAmongAnswered": 0.6666666666666666
},
"keywordBaseline": {
"count": 40,
"correct": 24,
"accuracy": 0.6,
"confusionMatrix": {
"bug": {
"bug": 3,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 5,
"rejected": 0
},
"feature": {
"bug": 0,
"feature": 5,
"documentation": 0,
"question": 0,
"unsure": 3,
"rejected": 0
},
"documentation": {
"bug": 0,
"feature": 1,
"documentation": 5,
"question": 0,
"unsure": 2,
"rejected": 0
},
"question": {
"bug": 0,
"feature": 1,
"documentation": 0,
"question": 6,
"unsure": 1,
"rejected": 0
},
"unsure": {
"bug": 0,
"feature": 0,
"documentation": 2,
"question": 1,
"unsure": 5,
"rejected": 0
},
"rejected": {
"bug": 0,
"feature": 0,
"documentation": 0,
"question": 0,
"unsure": 0,
"rejected": 0
}
},
"perLabel": {
"bug": {
"support": 8,
"precision": 1.0,
"recall": 0.375,
"f1": 0.5454545454545454
},
"feature": {
"support": 8,
"precision": 0.7142857142857143,
"recall": 0.625,
"f1": 0.6666666666666666
},
"documentation": {
"support": 8,
"precision": 0.7142857142857143,
"recall": 0.625,
"f1": 0.6666666666666666
},
"question": {
"support": 8,
"precision": 0.8571428571428571,
"recall": 0.75,
"f1": 0.7999999999999999
},
"unsure": {
"support": 8,
"precision": 0.3125,
"recall": 0.625,
"f1": 0.4166666666666667
}
},
"macroF1": 0.619090909090909,
"abstentionCount": 16,
"abstentionRate": 0.4,
"rejectedCount": 0,
"coverage": 0.6,
"accuracyAmongAnswered": 0.7916666666666666
},
"classificationFailures": [
{
"fixtureId": "heldout-002",
"title": "Search drops accented characters",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "bug",
"similarity": 0.31691886865612806,
"margin": 0.09499784081287282
},
{
"fixtureId": "heldout-003",
"title": "Next page returns the first page again",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "bug",
"similarity": 0.3209102342294865,
"margin": 0.08245770582829931
},
{
"fixtureId": "heldout-004",
"title": "Deleting a draft also removes the published copy",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "bug",
"similarity": 0.22369145427902837,
"margin": 0.019362608622608996
},
{
"fixtureId": "heldout-005",
"title": "Expired session gets an infinite redirect loop",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.20031679174045436,
"margin": 0.004790127325585897
},
{
"fixtureId": "heldout-006",
"title": "A valid configuration is rejected after upgrading",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.38372008443491556,
"margin": 0.07271489393168207
},
{
"fixtureId": "heldout-007",
"title": "Download produces a zero-byte archive",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "bug",
"similarity": 0.3654824885963447,
"margin": 0.16743207989225148
},
{
"fixtureId": "heldout-008",
"title": "Retry button does not retry",
"expected": "bug",
"observed": "unsure",
"rawSuggestedLabel": "bug",
"similarity": 0.23744552364420712,
"margin": 0.0548599137200812
},
{
"fixtureId": "heldout-009",
"title": "Let us pin favorite workspaces",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "feature",
"similarity": 0.30298730957483455,
"margin": 0.0809370624944632
},
{
"fixtureId": "heldout-010",
"title": "Provide a machine-readable command output option",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.15680882765931548,
"margin": 0.003234162286219505
},
{
"fixtureId": "heldout-011",
"title": "Add a configurable retention period",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "feature",
"similarity": 0.2704935731669698,
"margin": 0.1348431633305376
},
{
"fixtureId": "heldout-012",
"title": "Support importing newline-delimited JSON",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "feature",
"similarity": 0.258090359025198,
"margin": 0.12696420655872892
},
{
"fixtureId": "heldout-014",
"title": "Request a read-only project role",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "question",
"similarity": 0.1577232519718602,
"margin": 0.0011212976622386739
},
{
"fixtureId": "heldout-015",
"title": "Offer an undo operation for archiving",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "feature",
"similarity": 0.35220094685446207,
"margin": 0.18254807618538396
},
{
"fixtureId": "heldout-016",
"title": "Allow sorting by last successful run",
"expected": "feature",
"observed": "unsure",
"rawSuggestedLabel": "bug",
"similarity": 0.183475057487055,
"margin": 0.09443991962345641
},
{
"fixtureId": "heldout-019",
"title": "Explain the units of the timeout option",
"expected": "documentation",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.24593982761064337,
"margin": 0.028791182908548152
},
{
"fixtureId": "heldout-020",
"title": "Update the quickstart code sample",
"expected": "documentation",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.33435597124685906,
"margin": 0.16869413683050238
},
{
"fixtureId": "heldout-022",
"title": "The migration guide skips a rename",
"expected": "documentation",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.30095196430970306,
"margin": 0.03749232973938876
},
{
"fixtureId": "heldout-024",
"title": "Add a troubleshooting section to the manual",
"expected": "documentation",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.3206273157851509,
"margin": 0.01509125591616195
},
{
"fixtureId": "heldout-026",
"title": "Can existing jobs share one cache?",
"expected": "question",
"observed": "unsure",
"rawSuggestedLabel": "question",
"similarity": 0.14499380545436105,
"margin": 0.04747730482587681
},
{
"fixtureId": "heldout-027",
"title": "What does the queued status mean?",
"expected": "question",
"observed": "unsure",
"rawSuggestedLabel": "question",
"similarity": 0.24506617054350713,
"margin": 0.121169123828293
},
{
"fixtureId": "heldout-028",
"title": "Where can I find the logs?",
"expected": "question",
"observed": "unsure",
"rawSuggestedLabel": "documentation",
"similarity": 0.21972172315457267,
"margin": 0.010762111635235272
},
{
"fixtureId": "heldout-030",
"title": "Which setting limits concurrent workers?",
"expected": "question",
"observed": "unsure",
"rawSuggestedLabel": "question",
"similarity": 0.24336263306485956,
"margin": 0.0056780048828297125
},
{
"fixtureId": "heldout-031",
"title": "Does deleting a local copy affect the server?",
"expected": "question",
"observed": "unsure",
"rawSuggestedLabel": "question",
"similarity": 0.13517322815819105,
"margin": 0.06361004243122156
},
{
"fixtureId": "heldout-032",
"title": "How can I cancel a running export?",
"expected": "question",
"observed": "unsure",
"rawSuggestedLabel": "question",
"similarity": 0.33059763098099393,
"margin": 0.03460661000095605
},
{
"fixtureId": "heldout-040",
"title": "",
"expected": "unsure",
"observed": "rejected",
"rawSuggestedLabel": null,
"similarity": null,
"margin": null
}
],
"rejectedInputs": [
{
"fixtureId": "heldout-040",
"id": 1040,
"error": "Every issue needs a title of 1\u2013300 characters."
}
],
"duplicates": {
"items": 28,
"trueUniquePairs": 8,
"predictedUniquePairs": 8,
"truePositivePairs": 7,
"precision": 0.875,
"recall": 0.875,
"f1": 0.875,
"falsePositivePairs": [
[
2000,
3006
]
],
"missedPairs": [
[
2008,
2009
]
],
"top1PositiveAnchors": 16,
"top1Correct": 15,
"top1Recall": 0.9375,
"top1Details": [
{
"id": 2000,
"nearestCandidateId": 3006,
"similarity": 0.8467967829217281,
"correct": false
},
{
"id": 2001,
"nearestCandidateId": 2000,
"similarity": 0.8418473852003064,
"correct": true
},
{
"id": 2002,
"nearestCandidateId": 2003,
"similarity": 0.8772852109564382,
"correct": true
},
{
"id": 2003,
"nearestCandidateId": 2002,
"similarity": 0.8772852109564382,
"correct": true
},
{
"id": 2004,
"nearestCandidateId": 2005,
"similarity": 0.8968869256682411,
"correct": true
},
{
"id": 2005,
"nearestCandidateId": 2004,
"similarity": 0.8968869256682411,
"correct": true
},
{
"id": 2006,
"nearestCandidateId": 2007,
"similarity": 0.85970670731982,
"correct": true
},
{
"id": 2007,
"nearestCandidateId": 2006,
"similarity": 0.85970670731982,
"correct": true
},
{
"id": 2008,
"nearestCandidateId": 2009,
"similarity": 0.7246551750429989,
"correct": true
},
{
"id": 2009,
"nearestCandidateId": 2008,
"similarity": 0.7246551750429989,
"correct": true
},
{
"id": 2010,
"nearestCandidateId": 2011,
"similarity": 0.8888269888733635,
"correct": true
},
{
"id": 2011,
"nearestCandidateId": 2010,
"similarity": 0.8888269888733635,
"correct": true
},
{
"id": 2012,
"nearestCandidateId": 2013,
"similarity": 1,
"correct": true
},
{
"id": 2013,
"nearestCandidateId": 2012,
"similarity": 1,
"correct": true
},
{
"id": 2014,
"nearestCandidateId": 2015,
"similarity": 1,
"correct": true
},
{
"id": 2015,
"nearestCandidateId": 2014,
"similarity": 1,
"correct": true
}
],
"invalidCandidates": [],
"threshold": 0.83,
"interpretation": "Threshold suggestions point to an earlier item; nearest-candidate retrieval is across all other items. Scores are cosine similarity, not probabilities or proof."
},
"timingMs": {
"modelLoadMs": 125.48320000000001,
"referenceEmbeddingMs": 49.205917,
"classificationWallMs": 105.320741,
"duplicateWallMs": 76.24322299999994,
"warmRepeatWallMs": 107.59672
},
"processPeakRssKiB": 261600,
"nodeVersion": "v24.19.0",
"realVectorEvidence": {
"count": 2,
"dimensions": 384,
"finite": true,
"firstSixPerVector": [
[
-0.0038394939620047808,
-0.05189432203769684,
0.02111446112394333,
-0.043545182794332504,
0.08614278584718704,
-0.09509851038455963
],
[
0.03972547873854637,
0.033752620220184326,
0.030181316658854485,
0.04185470938682556,
-0.013174621388316154,
-0.08983414620161057
]
],
"vectorsDiffer": true
},
"warmRepeatPredictionsIdentical": true,
"sourceFreeze": {
"frozenAtUtc": "2026-10-19T22:35:20.411012+00:00",
"files": {
"package.json": "d738b7a364bd3d70a97046719ac87077601dd8b0d39f548d7ca2eb22795368c2",
"package-lock.json": "cee984f4133cc4df05633d618327db162be4335c94ee44f8ca10557bfc8518f8",
"src/core.mjs": "0090c8143ae3a4fe53194a564457d99983025f5dc7b009ef9e4f28f902709a9b",
"src/model.mjs": "7273332e105fcb43cd9c5bbe378fa63138d05da897cfb948f010e09a8f6c2351",
"src/server.mjs": "13832e0dfbca254cd346da388c9bb2077248251258927b74627c6e3d8b36231a",
"src/workflow.mjs": "a5771dab8a7f234b55e475b4fd1eb72e441bef91d846f7367c60079cd9532697",
"data/reference.json": "249505ab8a46972bc515e4427f8738a4314446488efd8499bbf2f4cc360b70d4",
"scripts/triage.mjs": "91a6856d2a53d9f6212a4430dcdf9018d9a2c3a279d85389bc2da49788b1433d",
"models/Xenova/all-MiniLM-L6-v2/onnx/model_quantized.onnx": "afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1",
"models/Xenova/all-MiniLM-L6-v2/config.json": "7135149f7cffa1a573466c6e4d8423ed73b62fd2332c575bf738a0d033f70df7",
"models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json": "9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3"
}
}
}