mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-10-21 22:01:21 +00:00
84c7ffad18
The embedding model is now chosen in the last onboarding step and fixed for the whole library. Skipping keeps bge-small, which ships with the app and can't be deleted. Changing the model later is not built. - An embedding_indexes row records the model's spec and its vector table, and each document records which index embedded it. Migration 0021 stamps existing libraries with bge-small. Routes that queue embedding work return 409 embedding_not_chosen until a model is chosen. - Ingest, Studio and search embed through the active index; the old worker embedding module is removed. Workers wait for the schema, and documents interrupted when the app closed are marked failed. - Curated embedders (bge-small, granite-97m, granite-311m) live in the local manifest, run by a new onnxruntime engine. EMBEDDING is a catalog type, never a selection slot. - Hugging Face search finds ONNX embedders. Each download is checked: a probe sets its width, and a ten-question search check must pass. Small tokenizer.json files that Hugging Face lists without a checksum are hashed at the pinned commit. Results use the GGUF search's shapes. - Onboarding renders the embedding step with the same component and shared list as the other steps; only its choice hook differs. Use marks the choice and Finish sends it. Bundled models have no Delete. - Settings has a read-only Embedding section. - ADRs 0036 and 0037, a new embedding architecture doc, and updates to the docs it touches. New strings are translated into every language.
56 lines
1.9 KiB
Python
56 lines
1.9 KiB
Python
"""Download the bundled embedding model for development, CI, and packaging.
|
|
|
|
`uv run scripts/fetch_embedding_model.py` places it where the app reads it in
|
|
development; pass a models root (`... models`) to stage it for an installer,
|
|
which electron-builder then copies into resources/models.
|
|
|
|
Fetched at bge's pinned revision and checked against its hashes: every existing
|
|
library was built with that exact file, so a different one is refused, not used.
|
|
"""
|
|
|
|
import hashlib
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
from modules.embedding.bundled import BGE, bundled_dir
|
|
|
|
BASE = "https://huggingface.co"
|
|
|
|
|
|
def fetch(into: Path) -> None:
|
|
into.mkdir(parents=True, exist_ok=True)
|
|
with httpx.Client(follow_redirects=True, timeout=120.0) as client:
|
|
for pinned in (BGE.weights, BGE.tokenizer):
|
|
target = into / pinned.path
|
|
if not target.exists():
|
|
print(f"get {pinned.path}")
|
|
url = f"{BASE}/{BGE.repo}/resolve/{BGE.revision}/{pinned.path}"
|
|
with client.stream("GET", url) as reply:
|
|
reply.raise_for_status()
|
|
with target.open("wb") as sink:
|
|
for block in reply.iter_bytes():
|
|
sink.write(block)
|
|
if _sha256(target) != pinned.sha256:
|
|
target.unlink()
|
|
raise SystemExit(f"{pinned.path} does not match bge's pinned hash")
|
|
print(f"have {target}")
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for block in iter(lambda: source.read(1 << 20), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def main() -> int:
|
|
# Optional models root for staging an installer; defaults to the dev location.
|
|
fetch(Path(sys.argv[1]) / BGE.id if len(sys.argv) > 1 else bundled_dir())
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|