prototype/scripts/download_model.py

"""Download a fixed, non-executable open model from Hugging Face. No auth or custom code."""
import hashlib, json, pathlib, urllib.request
ROOT = pathlib.Path(__file__).resolve().parents[1]
REPO = 'Xenova/all-MiniLM-L6-v2'
REVISION = '751bff37182d3f1213fa05d7196b954e230abad9'
FILES = ['config.json', 'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.txt', 'onnx/model_quantized.onnx', 'README.md']
# Frozen known bytes from the pinned revision. Never learn trust from a cache.
EXPECTED = {
    'config.json': (650, '7135149f7cffa1a573466c6e4d8423ed73b62fd2332c575bf738a0d033f70df7'),
    'tokenizer.json': (711661, 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0'),
    'tokenizer_config.json': (366, '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3'),
    'special_tokens_map.json': (125, 'b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3'),
    'vocab.txt': (231508, '07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3'),
    'onnx/model_quantized.onnx': (22972370, 'afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1'),
    'README.md': (1767, '63ea99bf681a2e9eda4f6a537d5ed8fda95d1677111656da37e9cfd080c3af02'),
}
LIMIT = 40 * 1024 * 1024
TOTAL_LIMIT = 48 * 1024 * 1024

def read_verified(path, url, expected_bytes, expected_hash, opener=urllib.request.urlopen):
    """Verify both existing and downloaded data before trusting or saving it."""
    downloaded = not path.exists()
    if downloaded:
        with opener(url, timeout=180) as response:
            size = response.headers.get('Content-Length')
            if size and int(size) > LIMIT:
                raise ValueError('Model artifact exceeds download limit')
            data = response.read(LIMIT + 1)
    else:
        with path.open('rb') as cached:
            data = cached.read(LIMIT + 1)
    if len(data) > LIMIT:
        raise ValueError('Model artifact exceeds download limit')
    if len(data) != expected_bytes or hashlib.sha256(data).hexdigest() != expected_hash:
        raise ValueError(f'Pinned artifact integrity mismatch: {path.name}')
    if downloaded:
        path.parent.mkdir(parents=True, exist_ok=True)
        temporary = path.with_suffix(path.suffix + '.part')
        temporary.write_bytes(data)
        temporary.replace(path)
    return data

def main():
    manifest = {'repo': REPO, 'revision': REVISION, 'source': f'https://huggingface.co/{REPO}/tree/{REVISION}', 'license': 'Apache-2.0', 'upstream': 'sentence-transformers/all-MiniLM-L6-v2', 'files': []}
    total = 0
    for name in FILES:
        path = ROOT / 'models' / REPO / name
        url = f'https://huggingface.co/{REPO}/resolve/{REVISION}/{name}'
        expected_bytes, expected_hash = EXPECTED[name]
        data = read_verified(path, url, expected_bytes, expected_hash)
        total += len(data)
        if total > TOTAL_LIMIT: raise ValueError('Total model download exceeds limit')
        manifest['files'].append({'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'source': url})
        print(name, len(data), flush=True)
    manifest['totalBytes'] = total
    (ROOT / 'models' / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
    print('Model ready; inference is configured local-only.', flush=True)
if __name__ == '__main__': main()