fix(models): keep architecture metadata when config.json fetch misses (#963)

The 2026-08-28 weekly scrape failed to fetch config.json for ~1,700 models
and overwrote known head/layer counts with null, which broke macOS CI
(test_mamba_name_does_not_erase_hybrid_attention_head_kv). Restore the
prior values, refuse to ship a scrape that has to rescue more than 25
models, and gate the weekly job on that invariant.

.github/workflows/weekly-model-update.yml

@@ -53,11 +53,14 @@ jobs:
       - name: Validate generated JSON
         run: python3 -m json.tool llmfit-core/data/hf_models.json > /dev/null
 
+      - name: Guard architecture metadata merge
+        run: python3 scripts/test_preserve_catalog_metadata.py
+
       - name: Set up Rust
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
 
-      - name: Validate against JSON Schema
-        run: cargo test -p llmfit-core hf_models_match_schema
+      - name: Validate catalog schema and known hybrid KV
+        run: cargo test -p llmfit-core hf_models_
 
       - name: Check for changes
         id: changes

llmfit-core/tests/schema_validation.rs

@@ -42,3 +42,37 @@ fn hf_models_match_schema() {
 
     validate_file(&schema, "data/hf_models.json");
 }
+
+#[test]
+fn hf_models_keep_known_hybrid_attention_heads() {
+    // Regression for the 2026-08-28 weekly scrape: a missed config.json fetch
+    // nullified architecture fields, and "mamba" in the name then erased KV.
+    // Weekly CI runs `hf_models_*`; keep this name so it gates the refresh PR.
+    let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/hf_models.json");
+    let text = std::fs::read_to_string(&path)
+        .unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display()));
+    let data: Value = serde_json::from_str(&text)
+        .unwrap_or_else(|e| panic!("cannot parse {}: {e}", path.display()));
+    let models = data.as_array().expect("hf_models.json is an array");
+    let required = [(
+        "QwerkyAI/Qwerky-Optimized-Llama3.2-Mamba-0.2-3B-Instruct",
+        24u64,
+        8u64,
+    )];
+    for (name, attn, kv) in required {
+        let model = models
+            .iter()
+            .find(|m| m.get("name").and_then(|v| v.as_str()) == Some(name))
+            .unwrap_or_else(|| panic!("missing {name} in embedded catalog"));
+        assert_eq!(
+            model.get("num_attention_heads").and_then(|v| v.as_u64()),
+            Some(attn),
+            "{name} lost num_attention_heads"
+        );
+        assert_eq!(
+            model.get("num_key_value_heads").and_then(|v| v.as_u64()),
+            Some(kv),
+            "{name} lost num_key_value_heads"
+        );
+    }
+}

scripts/scrape_hf_models.py

@@ -467,6 +467,61 @@ def estimate_vram(total_params: int, quant: str) -> float:
     return round(max(vram_gb, 0.5), 1)
 
 
+
+# Architecture fields filled from config.json. A failed fetch returns all-null
+# and used to overwrite good catalog values on the weekly merge. Keep the old
+# numbers when the fresh scrape did not produce them.
+ARCH_METADATA_KEYS = (
+    "num_hidden_layers",
+    "num_attention_heads",
+    "num_key_value_heads",
+    "head_dim",
+    "hidden_size",
+    "vocab_size",
+    "moe_intermediate_size",
+    "shared_expert_intermediate_size",
+)
+
+# A missed config.json fetch falls back to 4096. Restoring prior context is
+# only safe when architecture metadata was also dropped in the same pass.
+DEFAULT_CONTEXT_LENGTH = 4096
+
+# Mass config-fetch failure. One or two models can lose config.json for real;
+# 25+ in a single scrape is the 2026-08-28 incident (1,764 models).
+ARCH_METADATA_DROP_LIMIT = 25
+
+
+def preserve_existing_metadata(old_model: dict, fresh_model: dict) -> list[str]:
+    """Copy catalog fields the fresh scrape dropped. Returns restored keys.
+
+    Mutates ``fresh_model``. Used both to keep the weekly merge additive and
+    to count how many models needed an architecture-metadata rescue.
+    """
+    restored: list[str] = []
+    if old_model.get("license") and not fresh_model.get("license"):
+        fresh_model["license"] = old_model["license"]
+        restored.append("license")
+    if old_model.get("gguf_sources") and not fresh_model.get("gguf_sources"):
+        fresh_model["gguf_sources"] = old_model["gguf_sources"]
+        restored.append("gguf_sources")
+    for key in ("hf_downloads", "hf_likes", "release_date", "languages"):
+        if old_model.get(key) and not fresh_model.get(key):
+            fresh_model[key] = old_model[key]
+            restored.append(key)
+    for key in ARCH_METADATA_KEYS:
+        if old_model.get(key) is not None and fresh_model.get(key) is None:
+            fresh_model[key] = old_model[key]
+            restored.append(key)
+    if (
+        fresh_model.get("context_length") == DEFAULT_CONTEXT_LENGTH
+        and old_model.get("context_length") not in (None, DEFAULT_CONTEXT_LENGTH)
+        and "num_attention_heads" in restored
+    ):
+        fresh_model["context_length"] = old_model["context_length"]
+        restored.append("context_length")
+    return restored
+
+
 def extract_arch_metadata(config: dict | None) -> dict:
     """Extract architecture fields for precise KV cache and MoE speed estimation.
 
@@ -2972,6 +3027,7 @@ def main():
     existing_count = 0
     retained_count = 0
     updated_count = 0
+    arch_heads_restored = 0
     for output_path in output_paths:
         if os.path.exists(output_path):
             try:
@@ -2982,16 +3038,9 @@ def main():
                     name = old_model.get("name", "")
                     if name in fresh_by_name:
                         fresh_model = fresh_by_name[name]
-                        if old_model.get("license") and not fresh_model.get("license"):
-                            fresh_model["license"] = old_model["license"]
-                        if old_model.get("gguf_sources") and not fresh_model.get("gguf_sources"):
-                            fresh_model["gguf_sources"] = old_model["gguf_sources"]
-                        # Fallback stubs and trending listings carry no
-                        # popularity/date/language metadata — never let them
-                        # clobber real values from a previous scrape.
-                        for key in ("hf_downloads", "hf_likes", "release_date", "languages"):
-                            if old_model.get(key) and not fresh_model.get(key):
-                                fresh_model[key] = old_model[key]
+                        restored = preserve_existing_metadata(old_model, fresh_model)
+                        if "num_attention_heads" in restored:
+                            arch_heads_restored += 1
                         updated_count += 1
                     elif name:
                         # Historical model not in current scrape — keep it
@@ -3006,6 +3055,15 @@ def main():
     if existing_count:
         print(f"\nMerged with existing database ({existing_count} models):")
         print(f"  Updated: {updated_count}, Retained historical: {retained_count}")
+        print(f"  Architecture heads restored from prior catalog: {arch_heads_restored}")
+        if arch_heads_restored > ARCH_METADATA_DROP_LIMIT:
+            print(
+                f"ERROR: config.json fetch dropped attention-head metadata for "
+                f"{arch_heads_restored} models (limit {ARCH_METADATA_DROP_LIMIT}). "
+                f"Prior values were kept, but refusing to ship this scrape so the "
+                f"mass drop cannot land quietly again."
+            )
+            sys.exit(1)
 
     # Keep additive/retained entries on the current schema even if they were
     # produced by an older scraper version.

scripts/test_preserve_catalog_metadata.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Guard: weekly scrape must not silently wipe architecture metadata."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from scrape_hf_models import (  # noqa: E402
+    ARCH_METADATA_DROP_LIMIT,
+    preserve_existing_metadata,
+)
+
+
+def test_preserves_architecture_when_config_fetch_misses():
+    old = {
+        "license": "mit",
+        "num_attention_heads": 24,
+        "num_key_value_heads": 8,
+        "num_hidden_layers": 28,
+        "context_length": 4194304,
+        "hf_downloads": 89,
+    }
+    fresh = {
+        "license": None,
+        "num_attention_heads": None,
+        "num_key_value_heads": None,
+        "num_hidden_layers": None,
+        "context_length": 4096,
+        "hf_downloads": 0,
+    }
+    restored = preserve_existing_metadata(old, fresh)
+    assert fresh["num_attention_heads"] == 24, restored
+    assert fresh["num_key_value_heads"] == 8, restored
+    assert fresh["num_hidden_layers"] == 28, restored
+    assert fresh["context_length"] == 4194304, restored
+    assert fresh["license"] == "mit", restored
+    assert "num_attention_heads" in restored
+
+
+def test_does_not_invent_heads_the_catalog_never_had():
+    old = {"num_attention_heads": None, "context_length": 2048}
+    fresh = {"num_attention_heads": None, "context_length": 4096}
+    preserve_existing_metadata(old, fresh)
+    assert fresh["num_attention_heads"] is None
+    assert fresh["context_length"] == 4096
+
+
+def test_mass_drop_limit_would_have_caught_2026_08_28():
+    assert ARCH_METADATA_DROP_LIMIT < 1764
+
+
+if __name__ == "__main__":
+    tests = [
+        test_preserves_architecture_when_config_fetch_misses,
+        test_does_not_invent_heads_the_catalog_never_had,
+        test_mass_drop_limit_would_have_caught_2026_08_28,
+    ]
+    for fn in tests:
+        fn()
+        print(f"ok  {fn.__name__}")
+    print(f"{len(tests)} passed")