diff --git a/.github/workflows/weekly-model-update.yml b/.github/workflows/weekly-model-update.yml index 5b743c3d..116264f8 100644 --- a/.github/workflows/weekly-model-update.yml +++ b/.github/workflows/weekly-model-update.yml @@ -27,15 +27,17 @@ jobs: with: python-version: '3.12' - # Persist the GGUF source-probe cache (gitignored) between runs so the - # weekly scrape only re-checks entries past their max age instead of - # probing all ~10k models cold every time. The run_id key never matches - # exactly, so a fresh cache is uploaded after every run; restore-keys - # picks up the most recent one. - - name: Restore GGUF source cache + # Persist the HF probe caches (GGUF sources, release dates; gitignored) + # between runs so the weekly scrape only re-checks entries past their + # max age instead of probing all ~10k models cold every time. The + # run_id key never matches exactly, so a fresh cache is uploaded after + # every run; restore-keys picks up the most recent one. + - name: Restore HF probe caches uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: data/gguf_sources_cache.json + path: | + data/gguf_sources_cache.json + data/release_date_cache.json key: gguf-sources-cache-${{ github.run_id }} restore-keys: | gguf-sources-cache- @@ -61,11 +63,13 @@ jobs: # weeks left no cache (GitHub evicts after 7 unused days) and the # 2026-09-19 run probed ~12k models cold until the timeout killed it. # The scraper checkpoints the file as it probes. - - name: Save GGUF source cache - if: always() && hashFiles('data/gguf_sources_cache.json') != '' + - name: Save HF probe caches + if: always() && hashFiles('data/gguf_sources_cache.json', 'data/release_date_cache.json') != '' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: data/gguf_sources_cache.json + path: | + data/gguf_sources_cache.json + data/release_date_cache.json key: gguf-sources-cache-${{ github.run_id }} - name: Validate generated JSON diff --git a/.gitignore b/.gitignore index 293cb0b8..5108ae4f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /target llmfit-desktop/gen/schemas/* data/gguf_sources_cache.json +data/release_date_cache.json data/hf_models.json.backup.* llmfit-web/node_modules/ llmfit-web/dist/ diff --git a/scripts/scrape_hf_models.py b/scripts/scrape_hf_models.py index 1adcab1a..16c6987e 100644 --- a/scripts/scrape_hf_models.py +++ b/scripts/scrape_hf_models.py @@ -1574,8 +1574,9 @@ def revalidation_priority(model: dict) -> int | None: return 0 # packed parameter count nothing has been able to correct if (model.get("context_length") or 0) > SUSPECT_CONTEXT_LENGTH: return 1 - if not model.get("release_date"): - return 2 + # A missing release_date is not a reason to re-fetch the whole record: + # backfill_release_dates() fills it at one request per model, with its + # own cache, and the two passes do not share bookkeeping. return None @@ -2002,6 +2003,160 @@ def enrich_gguf_sources(models: list[dict], threads: int = 1, return enriched +# --------------------------------------------------------------------------- +# Release-date backfill (#176) +# --------------------------------------------------------------------------- +# The additive merge keeps historical entries as they are, so a model that +# entered the catalog without a release_date never gets one: 6,978 of the +# 15,007 entries in the 2026-09-19 catalog, all but 12 of them outside the +# curated list, because the listing did not expand createdAt. The listing +# now does, and this pass pays down the existing debt a bounded slice per +# run, through the model endpoint with `expand[]=createdAt` (id and date +# only, one request per model). +# +# Results are cached in RELEASE_DATE_CACHE_FILE so a killed run does not +# re-ask, and so a repo with no date (gone, or HF has none) is not asked +# again every week: negatives are re-checked after GGUF_CACHE_MAX_AGE_DAYS, +# the same cadence as the GGUF source cache. Dates are never guessed: no +# createdAt means None. + +RELEASE_DATE_CACHE_FILE = os.path.join( + os.path.dirname(__file__), "..", "data", "release_date_cache.json" +) +# Save the cache every N lookups so a job killed mid-backfill keeps its work. +RELEASE_DATE_CACHE_FLUSH_EVERY = 500 + + +def _load_release_date_cache() -> dict: + try: + with open(RELEASE_DATE_CACHE_FILE) as f: + data = json.load(f) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def _save_release_date_cache(cache: dict) -> None: + # Written to the side and swapped in, so a job killed mid-write leaves + # the previous cache intact instead of a truncated file the next run + # would read as empty. + os.makedirs(os.path.dirname(RELEASE_DATE_CACHE_FILE), exist_ok=True) + tmp = RELEASE_DATE_CACHE_FILE + ".tmp" + with open(tmp, "w") as f: + json.dump(cache, f, indent=2, sort_keys=True) + os.replace(tmp, RELEASE_DATE_CACHE_FILE) + + +# Answers that settle the question: the repo does not exist or cannot be +# read (401, 403 or 404, which one depends on the token). Anything else says +# nothing about the repo and must not be remembered as "no date". +_NO_DATE_HTTP_CODES = frozenset({401, 403, 404}) + + +def fetch_release_date(repo_id: str) -> str | None: + """createdAt of a repo as YYYY-MM-DD, or None when HF has no date for it + (repo gone or private, or no createdAt in the response). + Everything else propagates, a 429 that survived the retries, a 5xx, a + network error, a body that is not JSON or not the expected shape: the + answer is unknown, not "no date", and the caller must not cache it.""" + url = f"{HF_API}/{repo_id}?expand[]=createdAt" + try: + with _hf_urlopen(url, timeout=15, kind="release_date") as resp: + info = json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + if e.code in _NO_DATE_HTTP_CODES: + return None + raise + # A 200 is only an answer when it has the documented shape: an object + # whose createdAt is absent, null or empty (no date), or an ISO + # timestamp. Anything else is raised as ValueError, never returned as + # "no date", and never allowed to escape a worker as AttributeError or + # TypeError. + if not isinstance(info, dict): + raise ValueError(f"{repo_id}: expected an object, got {type(info).__name__}") + created = info.get("createdAt") + if created is None or created == "": + return None + if not isinstance(created, str): + raise ValueError(f"{repo_id}: createdAt is not a string: {created!r}") + return date.fromisoformat(created[:10]).isoformat() + + +def backfill_release_dates(models: list[dict], limit: int, threads: int = 1) -> dict: + """Fill release_date on entries that have none. Mutates ``models``. + + Most-downloaded first, since that is where the date sort is looked at. + At most ``limit`` HF lookups per run; the rest waits for the next one. + Returns the counters it prints, for the tests. + """ + cache = _load_release_date_cache() + candidates = [m for m in models if not m.get("release_date")] + candidates.sort(key=lambda m: -(m.get("hf_downloads") or 0)) + + stats = { + "filled": 0, + "from_cache": 0, + "unknown": 0, + "unsettled": 0, + "deferred": 0, + "lookups": 0, + } + to_fetch: list[dict] = [] + for model in candidates: + entry = cache.get(model["name"]) + if entry and entry.get("release_date"): + model["release_date"] = entry["release_date"] + stats["filled"] += 1 + stats["from_cache"] += 1 + elif entry and _cache_entry_fresh(entry): + stats["unknown"] += 1 # known to have no date, re-checked on cache cadence + else: + to_fetch.append(model) + + stats["deferred"] = max(0, len(to_fetch) - limit) + to_fetch = to_fetch[:limit] + + def _lookup(model: dict) -> tuple[dict, str | None, bool]: + try: + return model, fetch_release_date(model["name"]), True + except (OSError, ValueError, http.client.HTTPException): + # 429 past the retries, a 5xx, network trouble, a truncated or + # non-JSON body, or a body of the wrong shape: unknown this run, + # left uncached so the next one asks. One repo never aborts the + # run. + return model, None, False + + # One worker is plain sequential; map() keeps the most-downloaded-first + # order either way, and the cache is only touched from this thread. + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, threads)) as pool: + for model, created, settled in pool.map(_lookup, to_fetch): + stats["lookups"] += 1 + if not settled: + stats["unsettled"] += 1 + continue + cache[model["name"]] = { + "release_date": created, + "checked": datetime.now(timezone.utc).isoformat(), + } + if created: + model["release_date"] = created + stats["filled"] += 1 + else: + stats["unknown"] += 1 + if stats["lookups"] % RELEASE_DATE_CACHE_FLUSH_EVERY == 0: + _save_release_date_cache(cache) + + _save_release_date_cache(cache) + print( + f"\nRelease dates: {stats['filled']} filled " + f"({stats['from_cache']} from cache, {stats['lookups']} lookups), " + f"{stats['unknown']} without a date on HF, " + f"{stats['unsettled']} unanswered (rate limited or HF error, next run), " + f"{stats['deferred']} beyond this run's limit of {limit}" + ) + return stats + + # --------------------------------------------------------------------------- # Auto-discovery from HuggingFace trending / most-downloaded # --------------------------------------------------------------------------- @@ -2074,7 +2229,8 @@ def _build_first_page_url(pipeline: str, sort: str, page_size: int) -> str: f"limit={page_size}&" f"expand[]=safetensors&" f"expand[]=config&" - f"expand[]=cardData" + f"expand[]=cardData&" + f"expand[]=createdAt" ) @@ -2528,10 +2684,19 @@ def main(): help="Number of worker threads for parallel model metadata scraping " "(default: 1, which preserves current sequential behavior)." ) + parser.add_argument( + "--date-backfill-limit", type=int, default=500, + help="Max HuggingFace lookups per run to fill release_date on catalog " + "entries that have none (default: 500, one HF api window, 0 " + "disables). Results are cached in data/release_date_cache.json " + "so a killed run resumes." + ) args = parser.parse_args() if args.threads < 1: parser.error("--threads must be >= 1") + if args.date_backfill_limit < 0: + parser.error("--date-backfill-limit must be >= 0") # Resolve auth token: CLI flag > HF_TOKEN > HUGGING_FACE_HUB_TOKEN global _hf_token @@ -3821,6 +3986,13 @@ def main(): print(f" {rate_limit_summary()}") sys.exit(1) + # Fill release dates the additive merge never revisits (#176). Runs after + # the guard so a scrape that is refused does not spend lookups. + if args.date_backfill_limit: + backfill_release_dates( + results, limit=args.date_backfill_limit, threads=args.threads + ) + # Keep additive/retained entries on the current schema even if they were # produced by an older scraper version. for model in results: diff --git a/scripts/test_preserve_catalog_metadata.py b/scripts/test_preserve_catalog_metadata.py index 26b46f88..493d8275 100644 --- a/scripts/test_preserve_catalog_metadata.py +++ b/scripts/test_preserve_catalog_metadata.py @@ -479,12 +479,13 @@ def test_revalidation_ranks_uncorrectable_packed_counts_first(): assert revalidation_priority(plain) is None -def test_revalidation_flags_suspect_context_and_missing_date(): +def test_revalidation_flags_suspect_context_but_not_a_missing_date(): base = {"name": "org/model", "format": "gguf", "hidden_size": 4096, "release_date": "2026-01-01", "context_length": 131072} assert revalidation_priority(base) is None assert revalidation_priority({**base, "context_length": 16777216}) == 1 - assert revalidation_priority({**base, "release_date": None}) == 2 + # The release-date backfill owns this gap at one request per model. + assert revalidation_priority({**base, "release_date": None}) is None def test_revalidation_selection_respects_budget_and_skips_fresh_entries(): @@ -499,10 +500,11 @@ def test_revalidation_selection_respects_budget_and_skips_fresh_entries(): ] fresh = {"d/rescraped-AWQ"} # Priority beats popularity; downloads order entries within a priority. + # A dateless entry is left to the release-date backfill, however popular. assert select_retained_for_revalidation(existing, fresh, 2) == [ "c/model-GPTQ", "b/model-AWQ"] assert select_retained_for_revalidation(existing, fresh, 10) == [ - "c/model-GPTQ", "b/model-AWQ", "a/dateless-popular"] + "c/model-GPTQ", "b/model-AWQ"] assert select_retained_for_revalidation(existing, fresh, 0) == [] @@ -792,6 +794,198 @@ def test_one_broken_repo_is_skipped_not_fatal(): shm._build_discovered_model = saved +# --- release_date backfill (#176) -------------------------------------------- + + +def _date_url(repo_id: str) -> str: + return f"https://huggingface.co/api/models/{repo_id}?expand[]=createdAt" + + +class _DateCacheInMemory: + """backfill_release_dates with the cache file replaced by a dict.""" + + def __init__(self, preloaded: dict | None = None): + self.cache = dict(preloaded or {}) + + def __enter__(self): + self._saved = (shm._load_release_date_cache, shm._save_release_date_cache) + shm._load_release_date_cache = lambda: dict(self.cache) + shm._save_release_date_cache = self._save + self._quiet = contextlib.redirect_stdout(io.StringIO()) + self._quiet.__enter__() + return self + + def _save(self, cache): + self.cache = dict(cache) + + def __exit__(self, *exc): + self._quiet.__exit__(*exc) + shm._load_release_date_cache, shm._save_release_date_cache = self._saved + + +def _catalog(): + return [ + {"name": "org/small", "release_date": None, "hf_downloads": 5}, + {"name": "org/big", "release_date": None, "hf_downloads": 50}, + {"name": "org/dated", "release_date": "2023-01-01", "hf_downloads": 999}, + {"name": "org/throttled", "release_date": None, "hf_downloads": 1}, + ] + + +def test_discovery_listing_expands_created_at(): + url = shm._build_first_page_url("text-generation", "downloads", 1000) + assert "expand[]=createdAt" in url + for kept in ("expand[]=safetensors", "expand[]=config", "expand[]=cardData"): + assert kept in url, url + + +def test_backfill_fills_dates_most_downloaded_first_and_never_guesses(): + models = _catalog() + script = [ + _FakeResponse( + b'{"id": "org/big", "createdAt": "2024-11-25T15:06:15.000Z"}', None + ), + _http_error(_date_url("org/small"), 401), # gone or private: no date + ] + [ + _http_error(_date_url("org/throttled"), 429, ratelimit='"api";r=0;t=3') + for _ in range(RATE_LIMIT_MAX_RETRIES + 1) + ] + with _FakeHF(script) as hf, _DateCacheInMemory() as store: + stats = shm.backfill_release_dates(models, limit=10, threads=1) + assert [u.split("/models/")[1].split("?")[0] for u in hf.requests[:2]] == [ + "org/big", + "org/small", + ] + assert models[1]["release_date"] == "2024-11-25" + assert models[0]["release_date"] is None + assert models[3]["release_date"] is None + assert models[2]["release_date"] == "2023-01-01" + assert stats["filled"] == 1 and stats["unknown"] == 1 and stats["unsettled"] == 1 + assert store.cache["org/big"]["release_date"] == "2024-11-25" + assert store.cache["org/small"]["release_date"] is None + assert "org/throttled" not in store.cache, store.cache + + +def test_backfill_respects_the_per_run_limit(): + models = _catalog() + script = [ + _FakeResponse( + b'{"id": "org/big", "createdAt": "2024-11-25T15:06:15.000Z"}', None + ), + ] + with _FakeHF(script) as hf, _DateCacheInMemory(): + stats = shm.backfill_release_dates(models, limit=1, threads=1) + assert len(hf.requests) == 1 + assert stats["filled"] == 1 and stats["deferred"] == 2 + assert models[0]["release_date"] is None + + +def test_backfill_uses_the_cache_before_asking_hf(): + models = _catalog() + preloaded = { + "org/big": { + "release_date": "2024-11-25", + "checked": "2026-09-01T00:00:00+00:00", + }, + "org/small": { + "release_date": None, + "checked": shm.datetime.now(shm.timezone.utc).isoformat(), + }, + } + script = [ + _FakeResponse( + b'{"id": "org/throttled", "createdAt": "2025-02-02T00:00:00.000Z"}', None + ), + ] + with _FakeHF(script) as hf, _DateCacheInMemory(preloaded): + stats = shm.backfill_release_dates(models, limit=10, threads=1) + assert len(hf.requests) == 1 and "org/throttled" in hf.requests[0] + assert models[1]["release_date"] == "2024-11-25" + assert models[0]["release_date"] is None + assert models[3]["release_date"] == "2025-02-02" + assert stats["from_cache"] == 1 and stats["unknown"] == 1 and stats["filled"] == 2 + + +def test_stale_negative_is_asked_again(): + models = [{"name": "org/small", "release_date": None, "hf_downloads": 5}] + preloaded = { + "org/small": {"release_date": None, "checked": "2026-01-01T00:00:00+00:00"}, + } + script = [ + _FakeResponse( + b'{"id": "org/small", "createdAt": "2026-03-03T00:00:00.000Z"}', None + ), + ] + with _FakeHF(script) as hf, _DateCacheInMemory(preloaded): + shm.backfill_release_dates(models, limit=10, threads=1) + assert len(hf.requests) == 1 + assert models[0]["release_date"] == "2026-03-03" + + +def test_a_repo_without_created_at_stays_null_and_is_remembered(): + models = [{"name": "org/nodate", "release_date": None, "hf_downloads": 5}] + with ( + _FakeHF([_FakeResponse(b'{"id": "org/nodate"}', None)]), + _DateCacheInMemory() as store, + ): + stats = shm.backfill_release_dates(models, limit=10, threads=1) + assert models[0]["release_date"] is None + assert stats["unknown"] == 1 and stats["filled"] == 0 + assert store.cache["org/nodate"]["release_date"] is None + + +def test_transient_errors_are_not_cached_as_no_date(): + models = [ + {"name": "org/flaky", "release_date": None, "hf_downloads": 9}, + {"name": "org/offline", "release_date": None, "hf_downloads": 5}, + {"name": "org/gone", "release_date": None, "hf_downloads": 1}, + ] + script = [ + _http_error(_date_url("org/flaky"), 503), + urllib.error.URLError("connection reset"), + _http_error(_date_url("org/gone"), 404), + ] + with _FakeHF(script), _DateCacheInMemory() as store: + stats = shm.backfill_release_dates(models, limit=10, threads=1) + assert all(m["release_date"] is None for m in models) + assert stats["unsettled"] == 2 and stats["unknown"] == 1 and stats["filled"] == 0 + # A 5xx or a dropped connection says nothing about the repo: ask again + # next run instead of remembering "no date" for GGUF_CACHE_MAX_AGE_DAYS. + assert "org/flaky" not in store.cache and "org/offline" not in store.cache + assert store.cache["org/gone"]["release_date"] is None + + +def test_malformed_success_responses_do_not_abort_the_run_or_get_cached(): + models = [ + {"name": "org/null-body", "release_date": None, "hf_downloads": 9}, + {"name": "org/list-body", "release_date": None, "hf_downloads": 8}, + {"name": "org/numeric-date", "release_date": None, "hf_downloads": 7}, + {"name": "org/zero-date", "release_date": None, "hf_downloads": 6}, + {"name": "org/garbage-date", "release_date": None, "hf_downloads": 5}, + {"name": "org/truncated", "release_date": None, "hf_downloads": 4}, + {"name": "org/fine", "release_date": None, "hf_downloads": 3}, + ] + script = [ + _FakeResponse(b"null", None), + _FakeResponse(b"[]", None), + _FakeResponse(b'{"id": "org/numeric-date", "createdAt": 1700000000}', None), + _FakeResponse(b'{"id": "org/zero-date", "createdAt": 0}', None), + _FakeResponse(b'{"id": "org/garbage-date", "createdAt": "yesterday"}', None), + http.client.IncompleteRead(b'{"id": "org/tru'), + _FakeResponse(b'{"id": "org/fine", "createdAt": "2025-05-05T00:00:00.000Z"}', None), + ] + with _FakeHF(script) as hf, _DateCacheInMemory() as store: + stats = shm.backfill_release_dates(models, limit=10, threads=1) + # Each odd answer is contained to its repository: the run reaches the + # last model instead of dying on the first AttributeError or TypeError, + # and none of them is remembered as "no date". + assert len(hf.requests) == 7 + assert models[-1]["release_date"] == "2025-05-05" + assert all(m["release_date"] is None for m in models[:-1]) + assert stats["filled"] == 1 and stats["unsettled"] == 6 and stats["unknown"] == 0 + assert set(store.cache) == {"org/fine"}, store.cache + + if __name__ == "__main__": tests = [ test_preserves_architecture_when_config_fetch_misses, @@ -818,7 +1012,7 @@ if __name__ == "__main__": test_arch_metadata_drops_unset_sentinels, test_param_estimate_sees_the_same_experts_as_detection, test_revalidation_ranks_uncorrectable_packed_counts_first, - test_revalidation_flags_suspect_context_and_missing_date, + test_revalidation_flags_suspect_context_but_not_a_missing_date, test_revalidation_selection_respects_budget_and_skips_fresh_entries, test_packed_count_is_rescued_by_the_architecture_estimate, test_hybrid_ssm_models_are_never_sized_by_the_estimator, @@ -835,6 +1029,14 @@ if __name__ == "__main__": test_a_failing_estimate_keeps_the_reported_count, test_malformed_text_config_does_not_abort, test_one_broken_repo_is_skipped_not_fatal, + test_discovery_listing_expands_created_at, + test_backfill_fills_dates_most_downloaded_first_and_never_guesses, + test_backfill_respects_the_per_run_limit, + test_backfill_uses_the_cache_before_asking_hf, + test_stale_negative_is_asked_again, + test_a_repo_without_created_at_stays_null_and_is_remembered, + test_transient_errors_are_not_cached_as_no_date, + test_malformed_success_responses_do_not_abort_the_run_or_get_cached, ] for fn in tests: fn()