From 8d93aa026982628cd23bb907e45e72a8b527ddc5 Mon Sep 17 00:00:00 2001
From: Alex Jones <1235925+AlexsJones@users.noreply.github.com>
Date: Mon, 3 Aug 2026 14:36:43 +0100
Subject: [PATCH] ci: make the weekly HF model scrape finish inside its timeout

The scrape has been killed at the job timeout on every scheduled run
since 2026-07-13: GGUF source provenance probing (63b2d3a) issues 5
sequential candidate checks per model across ~10k models (~6h), and CI
never engaged the two mitigations the scraper already ships:

- the --threads flag existed but the workflow ran single-threaded;
  now --threads 8 (~45-60 min cold, aggregate ~3.6 req/s stays polite)
- the persistent probe cache (data/gguf_sources_cache.json) is
  gitignored, so runners always started cold; now restored/saved via
  actions/cache
- the cache max age was 7 days against a 7-day cron, so every entry
  was expired by the next run; now 13 days, staggering re-checks
  across alternating weeks

Job timeout raised to 120 min for cold-cache headroom.
---
 .github/workflows/weekly-model-update.yml | 20 ++++++++++++++++++--
 scripts/scrape_hf_models.py               |  6 +++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/weekly-model-update.yml b/.github/workflows/weekly-model-update.yml
index 5de621ed..ea274a41 100644
--- a/.github/workflows/weekly-model-update.yml
+++ b/.github/workflows/weekly-model-update.yml
@@ -16,7 +16,7 @@ permissions:
 jobs:
   update-models:
     runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 120
 
     steps:
       - name: Checkout
@@ -27,12 +27,28 @@ jobs:
         with:
           python-version: '3.12'
 
+      # Persist the GGUF source-probe cache (gitignored) between runs so the
+      # weekly scrape only re-checks entries past their max age instead of
+      # probing all ~10k models cold every time. The run_id key never matches
+      # exactly, so a fresh cache is uploaded after every run; restore-keys
+      # picks up the most recent one.
+      - name: Restore GGUF source cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: data/gguf_sources_cache.json
+          key: gguf-sources-cache-${{ github.run_id }}
+          restore-keys: |
+            gguf-sources-cache-
+
       # HF_TOKEN raises HuggingFace API rate limits and unlocks gated-model
       # metadata (the scraper warns-and-skips gated repos without it).
+      # --threads 8: sequential GGUF-source probing takes ~6h over ~10k
+      # models (5 candidates × 0.15s politeness sleep each) and was killed
+      # by the job timeout on every run from 2026-07-13 to 2026-08-03.
       - name: Refresh model database
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: python3 scripts/scrape_hf_models.py -n 5000
+        run: python3 scripts/scrape_hf_models.py -n 5000 --threads 8
 
       - name: Validate generated JSON
         run: python3 -m json.tool llmfit-core/data/hf_models.json > /dev/null
diff --git a/scripts/scrape_hf_models.py b/scripts/scrape_hf_models.py
index b725931d..a7a77ebe 100644
--- a/scripts/scrape_hf_models.py
+++ b/scripts/scrape_hf_models.py
@@ -1072,7 +1072,11 @@ def scrape_models_parallel(repo_ids: list[str], threads: int) -> tuple[list[dict
 GGUF_PROVIDERS = ["unsloth", "bartowski", "ggml-org", "TheBloke", "mradermacher"]
 
 GGUF_CACHE_FILE = os.path.join(os.path.dirname(__file__), "..", "data", "gguf_sources_cache.json")
-GGUF_CACHE_MAX_AGE_DAYS = 7  # Re-check repos older than this
+# Re-check repos older than this. Must exceed the weekly CI cadence (7 days)
+# or every cached entry is already expired by the next scheduled run and the
+# cache restored in CI is useless; 13 days staggers re-checks across runs
+# (roughly half the entries each week).
+GGUF_CACHE_MAX_AGE_DAYS = 13
 
 
 def _load_gguf_cache() -> dict: