mirror of
https://github.com/AlexsJones/llmfit.git
synced 2026-10-19 07:23:24 +00:00
ci: make the weekly HF model scrape finish inside its timeout
The scrape has been killed at the job timeout on every scheduled run
since 2026-07-13: GGUF source provenance probing (63b2d3a) issues 5
sequential candidate checks per model across ~10k models (~6h), and CI
never engaged the two mitigations the scraper already ships:
- the --threads flag existed but the workflow ran single-threaded;
now --threads 8 (~45-60 min cold, aggregate ~3.6 req/s stays polite)
- the persistent probe cache (data/gguf_sources_cache.json) is
gitignored, so runners always started cold; now restored/saved via
actions/cache
- the cache max age was 7 days against a 7-day cron, so every entry
was expired by the next run; now 13 days, staggering re-checks
across alternating weeks
Job timeout raised to 120 min for cold-cache headroom.
This commit is contained in:
parent
2209884211
commit
8d93aa0269
2 changed files with 23 additions and 3 deletions
20
.github/workflows/weekly-model-update.yml
vendored
20
.github/workflows/weekly-model-update.yml
vendored
|
|
@ -16,7 +16,7 @@ permissions:
|
|||
jobs:
|
||||
update-models:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
|
@ -27,12 +27,28 @@ jobs:
|
|||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
# Persist the GGUF source-probe cache (gitignored) between runs so the
|
||||
# weekly scrape only re-checks entries past their max age instead of
|
||||
# probing all ~10k models cold every time. The run_id key never matches
|
||||
# exactly, so a fresh cache is uploaded after every run; restore-keys
|
||||
# picks up the most recent one.
|
||||
- name: Restore GGUF source cache
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: data/gguf_sources_cache.json
|
||||
key: gguf-sources-cache-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
gguf-sources-cache-
|
||||
|
||||
# HF_TOKEN raises HuggingFace API rate limits and unlocks gated-model
|
||||
# metadata (the scraper warns-and-skips gated repos without it).
|
||||
# --threads 8: sequential GGUF-source probing takes ~6h over ~10k
|
||||
# models (5 candidates × 0.15s politeness sleep each) and was killed
|
||||
# by the job timeout on every run from 2026-07-13 to 2026-08-03.
|
||||
- name: Refresh model database
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: python3 scripts/scrape_hf_models.py -n 5000
|
||||
run: python3 scripts/scrape_hf_models.py -n 5000 --threads 8
|
||||
|
||||
- name: Validate generated JSON
|
||||
run: python3 -m json.tool llmfit-core/data/hf_models.json > /dev/null
|
||||
|
|
|
|||
|
|
@ -1072,7 +1072,11 @@ def scrape_models_parallel(repo_ids: list[str], threads: int) -> tuple[list[dict
|
|||
GGUF_PROVIDERS = ["unsloth", "bartowski", "ggml-org", "TheBloke", "mradermacher"]
|
||||
|
||||
GGUF_CACHE_FILE = os.path.join(os.path.dirname(__file__), "..", "data", "gguf_sources_cache.json")
|
||||
GGUF_CACHE_MAX_AGE_DAYS = 7 # Re-check repos older than this
|
||||
# Re-check repos older than this. Must exceed the weekly CI cadence (7 days)
|
||||
# or every cached entry is already expired by the next scheduled run and the
|
||||
# cache restored in CI is useless; 13 days staggers re-checks across runs
|
||||
# (roughly half the entries each week).
|
||||
GGUF_CACHE_MAX_AGE_DAYS = 13
|
||||
|
||||
|
||||
def _load_gguf_cache() -> dict:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue