"""Download a fixed, non-executable open model from Hugging Face. No auth or custom code.""" import hashlib, json, pathlib, urllib.request ROOT = pathlib.Path(__file__).resolve().parents[1] REPO = 'Xenova/all-MiniLM-L6-v2' REVISION = '751bff37182d3f1213fa05d7196b954e230abad9' FILES = ['config.json', 'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.txt', 'onnx/model_quantized.onnx', 'README.md'] # Frozen known bytes from the pinned revision. Never learn trust from a cache. EXPECTED = { 'config.json': (650, '7135149f7cffa1a573466c6e4d8423ed73b62fd2332c575bf738a0d033f70df7'), 'tokenizer.json': (711661, 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0'), 'tokenizer_config.json': (366, '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3'), 'special_tokens_map.json': (125, 'b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3'), 'vocab.txt': (231508, '07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3'), 'onnx/model_quantized.onnx': (22972370, 'afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1'), 'README.md': (1767, '63ea99bf681a2e9eda4f6a537d5ed8fda95d1677111656da37e9cfd080c3af02'), } LIMIT = 40 * 1024 * 1024 TOTAL_LIMIT = 48 * 1024 * 1024 def read_verified(path, url, expected_bytes, expected_hash, opener=urllib.request.urlopen): """Verify both existing and downloaded data before trusting or saving it.""" downloaded = not path.exists() if downloaded: with opener(url, timeout=180) as response: size = response.headers.get('Content-Length') if size and int(size) > LIMIT: raise ValueError('Model artifact exceeds download limit') data = response.read(LIMIT + 1) else: with path.open('rb') as cached: data = cached.read(LIMIT + 1) if len(data) > LIMIT: raise ValueError('Model artifact exceeds download limit') if len(data) != expected_bytes or hashlib.sha256(data).hexdigest() != expected_hash: raise ValueError(f'Pinned artifact integrity mismatch: {path.name}') if downloaded: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + '.part') temporary.write_bytes(data) temporary.replace(path) return data def main(): manifest = {'repo': REPO, 'revision': REVISION, 'source': f'https://huggingface.co/{REPO}/tree/{REVISION}', 'license': 'Apache-2.0', 'upstream': 'sentence-transformers/all-MiniLM-L6-v2', 'files': []} total = 0 for name in FILES: path = ROOT / 'models' / REPO / name url = f'https://huggingface.co/{REPO}/resolve/{REVISION}/{name}' expected_bytes, expected_hash = EXPECTED[name] data = read_verified(path, url, expected_bytes, expected_hash) total += len(data) if total > TOTAL_LIMIT: raise ValueError('Total model download exceeds limit') manifest['files'].append({'path': name, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'source': url}) print(name, len(data), flush=True) manifest['totalBytes'] = total (ROOT / 'models' / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') print('Model ready; inference is configured local-only.', flush=True) if __name__ == '__main__': main()