From 6884fb68325699a34fae7e93277673f11f118f56 Mon Sep 17 00:00:00 2001 From: Leo Borcherding Date: Sat, 10 Oct 2026 08:21:01 -0500 Subject: [PATCH] Unsloth Studio / Desktop (AMD, Windows): warn at startup when the AMD driver has the RDNA4 idle-eviction bug (#13208) * Unsloth Studio / Desktop (AMD, Windows): warn about AMD drivers with the RDNA4 idle-eviction bug Adrenalin 26.5.1 through PRO 26.9.1 let Windows power down an idle RDNA4 card and page out live allocations (ROCm/TheRock#7221), which freezes the whole system on multi-GPU hosts. 26.3.1 (32.0.22042.14002) is the last good build and 26.9.2 (32.0.32015.2008) the fix. The DirectX adapter read now keeps DriverVersion, the physical GPU inventory carries it, and a once-per-process check started from the lifespan (Windows only, daemon thread) flags an RDNA4 card whose driver sits strictly inside that range. It logs one warning and publishes gpu.driver_warning on /api/system; Settings > System resources shows it as a notice with a link to the issue, dismissible per driver version. Severity is critical when the host has more than one GPU. Warn only, nothing is blocked. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Anchor the AMD idle-eviction range at Adrenalin 26.5.1, the first affected build RDNA4's 26.1.1 to 26.3.1 drivers are numbered 32.0.23xxx, above the 32.0.22042.14002 26.3.1 package, so the old lower bound flagged every pre-26.5.1 RDNA4 driver, including 32.0.23033.1002, which TheRock#7221 measured as not evicting. * Trim comments in the AMD driver idle-eviction check * Read the installed AMD driver from the live adapter; open the notice's Details link in Desktop The DirectX registry keeps records of removed cards and the inventory pairs same-named records with live cards in LUID order, so a replaced R9700's old record could decide the warning. Win32_VideoController reports the installed DriverVersion per present adapter. The registry stays the cheap pre-check, so unaffected hosts still skip both WMI scans. target="_blank" has nowhere to go in the Tauri webview, so route the click through openLink. * Pair live AMD driver versions with inventory rows by the inventory's name rule --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han <23090290+danielhanchen@users.noreply.github.com> --- studio/backend/main.py | 9 + .../tests/test_amd_driver_idle_evict.py | 432 ++++++++++++++++++ studio/backend/utils/hardware/__init__.py | 6 + studio/backend/utils/hardware/hardware.py | 152 ++++++ .../stores/settings-panel-prefs-store.ts | 9 + .../features/settings/tabs/resources-tab.tsx | 66 ++- studio/frontend/src/i18n/locales/ar.ts | 4 + studio/frontend/src/i18n/locales/de.ts | 4 + studio/frontend/src/i18n/locales/en.ts | 4 + studio/frontend/src/i18n/locales/es.ts | 4 + studio/frontend/src/i18n/locales/fr.ts | 4 + studio/frontend/src/i18n/locales/he.ts | 4 + studio/frontend/src/i18n/locales/hi.ts | 4 + studio/frontend/src/i18n/locales/it.ts | 4 + studio/frontend/src/i18n/locales/ja.ts | 4 + studio/frontend/src/i18n/locales/ko.ts | 4 + studio/frontend/src/i18n/locales/pt-br.ts | 4 + studio/frontend/src/i18n/locales/ru.ts | 4 + studio/frontend/src/i18n/locales/sv.ts | 4 + studio/frontend/src/i18n/locales/zh-CN.ts | 4 + .../amd-driver-idle-evict-notice.test.ts | 85 ++++ .../tests/settings-panel-prefs.test.ts | 15 + 22 files changed, 829 insertions(+), 1 deletion(-) create mode 100644 studio/backend/tests/test_amd_driver_idle_evict.py create mode 100644 studio/frontend/tests/amd-driver-idle-evict-notice.test.ts diff --git a/studio/backend/main.py b/studio/backend/main.py index a78e2a4b0a..4b82f7c1ee 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -804,6 +804,13 @@ async def lifespan(app: FastAPI): except Exception: # noqa: BLE001 -- the first tool call probes instead _lifespan_log.warning("could not start the sandbox warm-up", exc_info = True) + # Windows AMD driver with the ROCm/TheRock#7221 idle-eviction bug: one log line, plus gpu.driver_warning. + try: + from utils.hardware.hardware import start_amd_driver_check + start_amd_driver_check() + except Exception: # noqa: BLE001 -- a warning, never a startup failure + _lifespan_log.debug("could not start the AMD driver check", exc_info = True) + try: from hub.services.models.account_access import adopt_unnamed_public_proofs from utils.hub_settings import operator_hf_endpoint @@ -2293,6 +2300,7 @@ def _get_cached_system_gpu_info( """Return training and inference GPU info with bounded live-probe churn.""" import time from utils.hardware import ( + amd_driver_warning_report, get_backend_visible_gpu_info, get_cross_vendor_inference_gpu_info, get_visible_gpu_utilization, @@ -2407,6 +2415,7 @@ def _get_cached_system_gpu_info( "vram_used_gb_aggregate": utilization_info.get("vram_used_gb_aggregate") if aggregate_basis_matches else None, + **amd_driver_warning_report(), } # Keep inference placement separate on train-capable hosts where a forced Vulkan llama.cpp bundle can diff --git a/studio/backend/tests/test_amd_driver_idle_evict.py b/studio/backend/tests/test_amd_driver_idle_evict.py new file mode 100644 index 0000000000..25a0083e49 --- /dev/null +++ b/studio/backend/tests/test_amd_driver_idle_evict.py @@ -0,0 +1,432 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""The startup notice for AMD Windows drivers with the ROCm/TheRock#7221 idle-eviction bug. + +Adrenalin 26.5.1 through PRO 26.9.1 page out live allocations on an idle RDNA4 card, which +freezes multi-GPU hosts; 26.5.1 (32.0.31007.1017) is the first bad build and 26.9.2 +(32.0.32015.2008) the fix. Every adapter and driver version here is pinned: nothing reads the +host's registry or WMI. +""" + +from __future__ import annotations + +import sys +import types + +import pytest + +import utils.hardware.hardware as hw +from utils.hardware import nvidia + +LAST_GOOD = "32.0.22042.14002" # Adrenalin 26.3.1 +FIRST_BAD = "32.0.31007.1017" # Adrenalin 26.5.1 +BROKEN = "32.0.31041.1004" # Adrenalin 26.8.1 +FIXED = "32.0.32015.2008" # Adrenalin 26.9.2 + + +def _amd( + name = "AMD Radeon AI PRO R9700", + gfx = "gfx1201", + driver = BROKEN, + **extra, +): + device = {"vendor": "amd", "name": name, "driver_version": driver, **extra} + if gfx: + device["gfx"] = gfx + return device + + +@pytest.fixture(autouse = True) +def _fresh_check(monkeypatch): + monkeypatch.setattr(hw, "_amd_driver_check_started", False) + monkeypatch.setattr(hw, "_amd_driver_notice", None) + monkeypatch.setattr(hw, "_physical_gpu_inventory_cache", None) + + +def _registry(monkeypatch, *records): + monkeypatch.setattr( + hw, + "_windows_amd_adapter_records_by_luid", + lambda *a, **k: {luid: dict(r) for luid, r in enumerate(records)}, + ) + + +@pytest.mark.parametrize( + "value,expected", + [ + # DirectX's REG_QWORD as read from an R9700 on Adrenalin 26.8.1. + (9007201289044972, (32, 0, 31041, 1004)), + ((32 << 48) | (22042 << 16) | 14002, (32, 0, 22042, 14002)), + ("32.0.32015.2008", (32, 0, 32015, 2008)), + (" 32.0.31041.1004 ", (32, 0, 31041, 1004)), + ("32.0.31041", None), + ("32.0.abc.1004", None), + ("32.0.31041.1004 beta", None), + ("", None), + (None, None), + (True, None), + (0, None), + (-1, None), + (1 << 64, None), + ], +) +def test_driver_version_parses_both_spellings(value, expected): + assert hw._parse_windows_driver_version(value) == expected + + +@pytest.mark.parametrize( + "driver,flagged", + [ + ("32.0.21001.9005", False), # older than 26.3.1 + (LAST_GOOD, False), + ("32.0.22042.14003", False), + ("32.0.23017.1001", False), # 26.1.1 + ("32.0.23033.1002", False), # 26.3.1 R9700 package, measured clean in TheRock#7221 + ("32.0.31007.1016", False), + (FIRST_BAD, True), + ("32.0.31036.15", True), # PRO 26.Q3 + ("32.0.31041.3013", True), # 26.9.1 + (BROKEN, True), + ("32.0.32015.2007", True), + (FIXED, False), + ("32.0.33010.1003", False), + ("33.0.10000.1", False), + ("not.a.driver.version", False), + (None, False), + ], +) +def test_only_drivers_inside_the_affected_range_are_flagged(driver, flagged): + notice = hw.amd_driver_idle_evict_notice([_amd(driver = driver)]) + assert (notice is not None) is flagged + if flagged: + assert notice["driver_version"] == driver + assert notice["severity"] == "warning" + assert notice["link"] == "https://github.com/ROCm/TheRock/issues/7221" + assert notice["devices"] == ["AMD Radeon AI PRO R9700"] + assert notice["message"] == ( + f"AMD driver {driver} has a known bug that can freeze Windows when an AMD GPU " + "sits idle, most often with more than one GPU. Update to Adrenalin 26.9.2 or later." + ) + + +def test_a_qwordshaped_driver_version_is_accepted_too(): + notice = hw.amd_driver_idle_evict_notice([_amd(driver = 9007201289044972)]) + assert notice is not None and notice["driver_version"] == BROKEN + + +def test_a_non_amd_card_is_never_flagged(): + nvidia_card = {"vendor": "nvidia", "name": "NVIDIA GeForce RTX 4090", "driver_version": BROKEN} + assert hw.amd_driver_idle_evict_notice([nvidia_card]) is None + + +@pytest.mark.parametrize( + "name,gfx,flagged", + [ + ("AMD Radeon RX 7900 XTX", "gfx1100", False), + ("AMD Radeon(TM) 8060S Graphics", "gfx1151", False), + ("AMD Radeon RX 9060 XT", "gfx1200", True), + ("AMD Radeon RX 9070 XT", "gfx1201", True), + # gfx1250 is Instinct, not RDNA4. + ("AMD Instinct MI350X", "gfx1250", False), + # A driver that wrote no AdapterFamily: the name decides. + ("AMD Radeon RX 9070", None, True), + ("AMD Radeon AI PRO R9700", None, True), + ("AMD Radeon RX 7800 XT", None, False), + ("AMD Radeon 780M Graphics", None, False), + ], +) +def test_only_rdna4_amd_cards_are_flagged(name, gfx, flagged): + notice = hw.amd_driver_idle_evict_notice([_amd(name = name, gfx = gfx)]) + assert (notice is not None) is flagged + + +def test_an_arch_from_the_registry_beats_the_name(): + # A gfx1100 record that happens to carry an RDNA4-looking name is still RDNA3. + assert hw.amd_driver_idle_evict_notice([_amd(name = "RX 9070 (rebadged)", gfx = "gfx1100")]) is None + + +def test_one_gpu_is_a_warning(): + assert hw.amd_driver_idle_evict_notice([_amd()])["severity"] == "warning" + + +@pytest.mark.parametrize( + "others", + [ + [_amd()], # 3x R9700 shape, cut to two + [_amd(name = "AMD Radeon RX 9070", gfx = "gfx1201")], # RX 9070 + R9700 + [_amd(name = "AMD Radeon(TM) Graphics", gfx = "gfx1036")], # Ryzen iGPU driving the display + [{"vendor": "nvidia", "name": "NVIDIA GeForce RTX 3060"}], + [{"vendor": "intel", "name": "Intel(R) UHD Graphics 770"}], + ], +) +def test_more_than_one_gpu_is_critical(others): + notice = hw.amd_driver_idle_evict_notice([_amd(), *others]) + assert notice["severity"] == "critical" + assert notice["gpu_count"] == 2 + + +def test_no_devices_is_no_notice(): + assert hw.amd_driver_idle_evict_notice([]) is None + + +class _InlineThread: + def __init__( + self, + target, + name = None, + daemon = None, + ): + self._target = target + + def start(self): + self._target() + + +def _live(monkeypatch, versions): + pairs = None if versions is None else list(versions.items()) + monkeypatch.setattr(hw, "_windows_live_amd_driver_versions", lambda: pairs) + + +def test_non_windows_is_a_no_op(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + + def _no_threads(*a, **k): + raise AssertionError("no thread off Windows") + + monkeypatch.setattr(hw.threading, "Thread", _no_threads) + monkeypatch.setattr( + hw, "get_physical_gpu_inventory", lambda **k: pytest.fail("no inventory off Windows") + ) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +def test_windows_check_logs_once_and_publishes(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + calls = [] + + def _inventory(**k): + calls.append(k) + return {"devices": [_amd(), _amd()]} + + monkeypatch.setattr(hw, "get_physical_gpu_inventory", _inventory) + _registry(monkeypatch, _amd()) + _live(monkeypatch, {"AMD Radeon AI PRO R9700": BROKEN}) + warnings = [] + monkeypatch.setattr(hw.logger, "warning", lambda msg, *args: warnings.append(msg % args)) + + hw.start_amd_driver_check() + hw.start_amd_driver_check() # once per process + + assert len(calls) == 1 + assert len(warnings) == 1 and BROKEN in warnings[0] and "TheRock/issues/7221" in warnings[0] + report = hw.amd_driver_warning_report() + assert report["driver_warning"]["severity"] == "critical" + assert report["driver_warning"]["driver_version"] == BROKEN + + +def test_a_fixed_driver_publishes_nothing(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + _registry(monkeypatch, _amd()) + _live(monkeypatch, {"AMD Radeon AI PRO R9700": FIXED}) + monkeypatch.setattr(hw, "get_physical_gpu_inventory", lambda **k: {"devices": [_amd()]}) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +@pytest.mark.parametrize( + "live,flagged", + [({"AMD Radeon AI PRO R9700": FIXED}, False), ({"AMD Radeon AI PRO R9700": BROKEN}, True)], +) +def test_the_live_adapter_decides_over_a_stale_same_named_record(monkeypatch, live, flagged): + # One of two identical R9700s was replaced: the registry holds both an old and a current + # record under one name, and only WMI knows which version is installed. + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + _registry(monkeypatch, _amd(driver = BROKEN), _amd(driver = FIXED)) + _live(monkeypatch, live) + monkeypatch.setattr( + hw, "get_physical_gpu_inventory", lambda **k: {"devices": [_amd(driver = BROKEN)]} + ) + hw.start_amd_driver_check() + assert bool(hw.amd_driver_warning_report()) is flagged + + +def test_live_names_match_like_the_inventory_join(monkeypatch): + # The registry row says "AMD Radeon AI PRO R9700", WMI a differently cased, longer name. + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + _registry(monkeypatch, _amd()) + _live(monkeypatch, {"AMD RADEON AI PRO R9700 32GB": BROKEN}) + monkeypatch.setattr(hw, "get_physical_gpu_inventory", lambda **k: {"devices": [_amd()]}) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report()["driver_warning"]["driver_version"] == BROKEN + + +def test_an_unanswered_live_scan_publishes_nothing(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + _registry(monkeypatch, _amd()) + _live(monkeypatch, None) + monkeypatch.setattr(hw, "get_physical_gpu_inventory", lambda **k: {"devices": [_amd()]}) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +@pytest.mark.parametrize( + "records", + [ + (), + (_amd(driver = FIXED),), + (_amd(name = "AMD Radeon RX 7900 XTX", gfx = "gfx1100"),), + ], +) +def test_no_affected_registry_record_skips_the_live_scan(monkeypatch, records): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + _registry(monkeypatch, *[{k: v for k, v in r.items() if k != "vendor"} for r in records]) + monkeypatch.setattr( + hw, "get_physical_gpu_inventory", lambda **k: pytest.fail("no WMI scan without a lead") + ) + monkeypatch.setattr( + hw, "_windows_live_amd_driver_versions", lambda: pytest.fail("no WMI scan without a lead") + ) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +def test_a_stale_registry_record_the_live_scan_drops_publishes_nothing(monkeypatch): + # A removed R9700 left its old record behind; the card in the box is an RX 7900 XTX. + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + _registry(monkeypatch, _amd()) + _live(monkeypatch, {"AMD Radeon RX 7900 XTX": BROKEN}) + monkeypatch.setattr( + hw, + "get_physical_gpu_inventory", + lambda **k: {"devices": [_amd(name = "AMD Radeon RX 7900 XTX", gfx = "gfx1100")]}, + ) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +def test_a_failing_inventory_never_raises(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr(hw.threading, "Thread", _InlineThread) + + def _boom(**k): + raise RuntimeError("WMI is gone") + + monkeypatch.setattr(hw, "get_physical_gpu_inventory", _boom) + _registry(monkeypatch, _amd()) + _live(monkeypatch, {"AMD Radeon AI PRO R9700": BROKEN}) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +def test_a_thread_that_will_not_start_never_raises(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + + def _cannot_start(*a, **k): + raise RuntimeError("can't start new thread") + + monkeypatch.setattr(hw.threading, "Thread", _cannot_start) + hw.start_amd_driver_check() + assert hw.amd_driver_warning_report() == {} + + +def _fake_winreg(subkeys): + mod = types.ModuleType("winreg") + mod.HKEY_LOCAL_MACHINE = object() + names = list(subkeys) + + class _Key: + def __init__(self, name): + self.name = name + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def open_key(parent, sub): + return _Key(None if parent is mod.HKEY_LOCAL_MACHINE else sub) + + def query_value_ex(key, value): + values = subkeys[key.name] + if value not in values: + raise FileNotFoundError(2, "value does not exist") + return (values[value], 0) + + mod.OpenKey = open_key + mod.QueryInfoKey = lambda key: (len(names), 0, 0) + mod.EnumKey = lambda key, index: names[index] + mod.QueryValueEx = query_value_ex + return mod + + +R9700_RECORD = { + "VendorId": 0x1002, + "AdapterLuid": 82776, + "Description": "AMD Radeon AI PRO R9700", + "AdapterFamily": "AMD_NAVI48:gfx1201", + "DriverVersion": 9007201289044972, +} + + +def test_registry_record_carries_the_driver_version(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setitem( + sys.modules, + "winreg", + _fake_winreg({"{cd12ca95-0000-0000-0000-000000000000}": R9700_RECORD}), + ) + assert hw._windows_amd_adapter_records_by_luid() == { + 82776: {"name": "AMD Radeon AI PRO R9700", "gfx": "gfx1201", "driver_version": BROKEN} + } + + +@pytest.mark.parametrize("driver", ["garbage", 0]) +def test_an_unreadable_driver_version_does_not_decline_the_map(monkeypatch, driver): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setitem( + sys.modules, + "winreg", + _fake_winreg( + {"{cd12ca95-0000-0000-0000-000000000000}": {**R9700_RECORD, "DriverVersion": driver}} + ), + ) + assert hw._windows_amd_adapter_records_by_luid() == { + 82776: {"name": "AMD Radeon AI PRO R9700", "gfx": "gfx1201"} + } + + +def test_live_scan_reads_amd_adapters_by_name(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + seen = {} + + def _run(argv, **kw): + seen["ps"] = argv[-1] + return types.SimpleNamespace( + returncode = 0, + stdout = f"AMD Radeon AI PRO R9700\t{BROKEN}\r\nAMD Radeon RX 9070 XT\t{BROKEN}\n\n", + ) + + monkeypatch.setattr(hw.subprocess, "run", _run) + assert hw._windows_live_amd_driver_versions() == [ + ("AMD Radeon AI PRO R9700", BROKEN), + ("AMD Radeon RX 9070 XT", BROKEN), + ] + assert "PCI\\VEN_1002*" in seen["ps"] + + +def test_a_failed_live_scan_is_none(monkeypatch): + monkeypatch.setattr(hw.platform, "system", lambda: "Windows") + monkeypatch.setattr( + hw.subprocess, "run", lambda *a, **k: types.SimpleNamespace(returncode = 1, stdout = "") + ) + assert hw._windows_live_amd_driver_versions() is None diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py index b45c12fa0a..a8b179eca9 100644 --- a/studio/backend/utils/hardware/__init__.py +++ b/studio/backend/utils/hardware/__init__.py @@ -78,6 +78,11 @@ def video_capability() -> dict: return _hardware.video_capability() +def amd_driver_warning_report() -> dict: + """Return the AMD driver idle-eviction notice, if the startup check flagged one.""" + return _hardware.amd_driver_warning_report() + + def get_torch_device_str() -> str: """Return the torch device string ("cuda", "xpu", "cpu") for the detected hardware.""" return _hardware.get_torch_device_str() @@ -94,6 +99,7 @@ __all__ = [ "get_device", "export_capability", "video_capability", + "amd_driver_warning_report", "is_apple_silicon", "clear_gpu_cache", "get_gpu_memory_info", diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 5ffc5f5716..88824c991a 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -3286,6 +3286,10 @@ def _windows_amd_adapter_records_or_none( family, _ = winreg.QueryValueEx(adapter_key, "AdapterFamily") except OSError: family = "" + try: + driver_version, _ = winreg.QueryValueEx(adapter_key, "DriverVersion") + except OSError: + driver_version = None dedicated_memory_bytes = 0 has_dedicated_memory = False for value_name in ("DedicatedVideoMemory", "DedicatedSystemMemory"): @@ -3307,6 +3311,9 @@ def _windows_amd_adapter_records_or_none( record["gfx"] = gfx if has_dedicated_memory: record["dedicated_memory_bytes"] = dedicated_memory_bytes + driver = _parse_windows_driver_version(driver_version) + if driver is not None: + record["driver_version"] = _format_driver_version(driver) by_luid[int(luid)] = record except Exception as e: logger.debug("DirectX adapter registry read declined: %s", e) @@ -3314,6 +3321,151 @@ def _windows_amd_adapter_records_or_none( return by_luid +# ROCm/TheRock#7221: Adrenalin 26.5.1 through PRO 26.9.1 let Windows power down an idle RDNA4 card and page out live HIP allocations, which freezes the whole system on multi-GPU hosts. 26.5.1 is the first bad build, 26.9.2 the fix; never a 26.3.1 bound, since RDNA4's clean 26.1.1-26.3.1 builds are 32.0.23xxx (gpuopen.com/version-table). +_AMD_IDLE_EVICT_FIRST_BAD_DRIVER = (32, 0, 31007, 1017) +_AMD_IDLE_EVICT_FIXED_DRIVER = (32, 0, 32015, 2008) +_AMD_IDLE_EVICT_ISSUE_URL = "https://github.com/ROCm/TheRock/issues/7221" +# Name fallback for a driver that wrote no AdapterFamily: RX 9060/9070 series and Radeon AI PRO R9600D/R9700. +_RDNA4_ADAPTER_NAME = re.compile(r"\bRX\s*90[67]0\b|\bR9[67]00D?\b", re.IGNORECASE) +_amd_driver_check_lock = threading.Lock() +_amd_driver_check_started = False +_amd_driver_notice: Optional[Dict[str, Any]] = None + + +def _parse_windows_driver_version(value: Any) -> Optional[tuple[int, int, int, int]]: + """DirectX stores DriverVersion as a REG_QWORD of four 16-bit fields; the INF spells it "32.0.A.B". Either form, or None.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + if not 0 < value < 1 << 64: + return None + return tuple((value >> shift) & 0xFFFF for shift in (48, 32, 16, 0)) + m = re.fullmatch(r"\s*(\d+)\.(\d+)\.(\d+)\.(\d+)\s*", str(value or "")) + return tuple(int(part) for part in m.groups()) if m else None + + +def _format_driver_version(version: tuple[int, ...]) -> str: + return ".".join(str(part) for part in version) + + +def _is_rdna4_adapter(device: Dict[str, Any]) -> bool: + gfx = str(device.get("gfx") or "").lower() + if gfx: + # gfx1250 is Instinct, so not a bare gfx12 prefix. + return gfx.startswith("gfx120") + return bool(_RDNA4_ADAPTER_NAME.search(str(device.get("name") or ""))) + + +def amd_driver_idle_evict_notice(devices: list[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """The TheRock#7221 notice for these physical inventory rows, or None. Severity is "critical" when the host has more than one GPU of any vendor, the shape that freezes Windows.""" + flagged = [] + for device in devices: + if device.get("vendor") != "amd" or not _is_rdna4_adapter(device): + continue + version = _parse_windows_driver_version(device.get("driver_version")) + if ( + version is not None + and _AMD_IDLE_EVICT_FIRST_BAD_DRIVER <= version < _AMD_IDLE_EVICT_FIXED_DRIVER + ): + flagged.append((device, _format_driver_version(version))) + if not flagged: + return None + driver_version = flagged[0][1] + return { + "message": ( + f"AMD driver {driver_version} has a known bug that can freeze Windows when an AMD " + "GPU sits idle, most often with more than one GPU. Update to Adrenalin 26.9.2 or later." + ), + "driver_version": driver_version, + "devices": [device.get("name") for device, _ in flagged], + "gpu_count": len(devices), + "severity": "critical" if len(devices) > 1 else "warning", + "link": _AMD_IDLE_EVICT_ISSUE_URL, + } + + +def _windows_live_amd_driver_versions() -> Optional[list[tuple[str, str]]]: + """(Name, DriverVersion) of each present AMD display adapter, or None when WMI could not say. Read live because the DirectX registry keeps records of removed cards, and the inventory pairs same-named records with live cards in LUID order.""" + if platform.system() != "Windows": + return None + try: + ps = ( + "$ErrorActionPreference='Stop';" + "(Get-CimInstance Win32_VideoController" + " | Where-Object { $_.PNPDeviceID -like 'PCI\\VEN_1002*' }" + ' | ForEach-Object { $_.Name + "`t" + $_.DriverVersion }) -join "`n"' + ) + r = subprocess.run( + ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 10, + **_hidden_console_kwargs(), + ) + except (OSError, subprocess.SubprocessError) as e: + logger.debug("Live AMD driver version scan failed: %s", e) + return None + if r.returncode != 0: + return None + versions = [] + for line in (r.stdout or "").splitlines(): + name, _, version = line.partition("\t") + if name.strip() and version.strip(): + versions.append((name.strip(), version.strip())) + return versions + + +def _check_amd_driver_idle_evict() -> None: + global _amd_driver_notice + try: + # The registry alone first: a host with no affected record never pays for the live WMI scans. + records = [{"vendor": "amd", **r} for r in _windows_amd_adapter_records_by_luid().values()] + if amd_driver_idle_evict_notice(records) is None: + return + live = _windows_live_amd_driver_versions() + if live is None: + return + rows = [] + for d in get_physical_gpu_inventory().get("devices") or []: + if d.get("vendor") == "amd": + # The inventory's own name join: rows carry the registry description, WMI the display name. + match = _claim_live_adapter(d.get("name"), [name for name, _ in live]) + d = {**d, "driver_version": live.pop(match)[1] if match is not None else None} + rows.append(d) + notice = amd_driver_idle_evict_notice(rows) + except Exception as e: + logger.debug("AMD driver idle-eviction check failed: %s", e) + return + _amd_driver_notice = notice + if notice is not None: + logger.warning("%s See %s", notice["message"], notice["link"]) + + +def start_amd_driver_check() -> None: + """Check the AMD driver against TheRock#7221 once per process, Windows only, on a daemon thread: the inventory it reads runs a WMI scan, and its result is cached for the next caller.""" + global _amd_driver_check_started + if platform.system() != "Windows": + return + with _amd_driver_check_lock: + if _amd_driver_check_started: + return + _amd_driver_check_started = True + try: + threading.Thread( + target = _check_amd_driver_idle_evict, name = "amd-driver-check", daemon = True + ).start() + except Exception as e: + logger.debug("Could not start the AMD driver check: %s", e) + + +def amd_driver_warning_report() -> Dict[str, Any]: + """``driver_warning`` for /api/system once the startup check has flagged this host's driver, else ``{}``.""" + notice = _amd_driver_notice + return {"driver_warning": dict(notice)} if notice else {} + + def _windows_rocm_shared_pool_host_gb_by_index(devices: list[Dict[str, Any]]) -> Dict[int, float]: """Map shared ROCm devices to the host-backed part of their Windows pool.""" if platform.system() != "Windows": diff --git a/studio/frontend/src/features/settings/stores/settings-panel-prefs-store.ts b/studio/frontend/src/features/settings/stores/settings-panel-prefs-store.ts index f64fb2bf32..a41484f9d8 100644 --- a/studio/frontend/src/features/settings/stores/settings-panel-prefs-store.ts +++ b/studio/frontend/src/features/settings/stores/settings-panel-prefs-store.ts @@ -42,6 +42,10 @@ export interface SettingsPanelPrefsState { resourcesLiveUpdates: boolean; setResourcesLiveUpdates: (enabled: boolean) => void; + // the driver version whose idle-eviction notice was dismissed, so a different bad driver shows again. + dismissedDriverWarning: string | null; + dismissDriverWarning: (driverVersion: string) => void; + fineTuneAction: FineTuneAction; setFineTuneAction: (action: FineTuneAction) => void; } @@ -97,6 +101,7 @@ function sanitize( typeof raw.resourcesLiveUpdates === "boolean" ? raw.resourcesLiveUpdates : true, + dismissedDriverWarning: text(raw.dismissedDriverWarning), fineTuneAction: oneOf(raw.fineTuneAction, FINE_TUNE_VALUES, "train"), }; } @@ -133,6 +138,10 @@ export const useSettingsPanelPrefsStore = create()( setResourcesLiveUpdates: (resourcesLiveUpdates) => set({ resourcesLiveUpdates }), + dismissedDriverWarning: null, + dismissDriverWarning: (dismissedDriverWarning) => + set({ dismissedDriverWarning }), + fineTuneAction: "train", setFineTuneAction: (fineTuneAction) => set({ fineTuneAction }), }), diff --git a/studio/frontend/src/features/settings/tabs/resources-tab.tsx b/studio/frontend/src/features/settings/tabs/resources-tab.tsx index 85e11a39aa..ea5c0f8650 100644 --- a/studio/frontend/src/features/settings/tabs/resources-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/resources-tab.tsx @@ -29,6 +29,7 @@ import { } from "@/hooks/use-system"; import { isTauri } from "@/lib/api-base"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { openLink } from "@/lib/open-link"; import { toast } from "@/lib/toast"; import { cn } from "@/lib/utils"; import { useT } from "@/i18n"; @@ -47,7 +48,7 @@ import { SettingsRow } from "../components/settings-row"; import { SettingsSection } from "../components/settings-section"; import { useMonitorOverlayStore } from "../stores/monitor-overlay-store"; import { useSettingsPanelPrefsStore } from "../stores/settings-panel-prefs-store"; -import { Copy01Icon } from "@hugeicons/core-free-icons"; +import { ArrowUpRight01Icon, Copy01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { FolderOpenIcon, LayersIcon } from "lucide-react"; @@ -237,9 +238,18 @@ interface GpuTorchMismatch { physical_count?: number; } +/** /api/system's `gpu.driver_warning`: a Windows AMD driver with the idle-eviction bug + * (ROCm/TheRock#7221). "critical" when the host has more than one GPU. */ +interface GpuDriverWarning { + driver_version?: string; + severity?: "warning" | "critical"; + link?: string; +} + type GpuPhysicalInventory = SystemGpuInfo & { physical_devices?: PhysicalGpuDevice[]; mismatch?: GpuTorchMismatch | null; + driver_warning?: GpuDriverWarning | null; }; export function ResourcesTab() { @@ -248,6 +258,12 @@ export function ResourcesTab() { const setLiveUpdates = useSettingsPanelPrefsStore( (s) => s.setResourcesLiveUpdates, ); + const dismissedDriverWarning = useSettingsPanelPrefsStore( + (s) => s.dismissedDriverWarning, + ); + const dismissDriverWarning = useSettingsPanelPrefsStore( + (s) => s.dismissDriverWarning, + ); const { isOpen, setIsOpen } = useMonitorOverlayStore(); // always fetch once: the switch is persisted now, so gating the hook on it // would leave a session that opens with it off reading zeros forever. @@ -494,6 +510,12 @@ export function ResourcesTab() { const physicalDevices = gpuMismatch ? (gpuInventory?.physical_devices ?? []) : []; + const rawDriverWarning = gpuInventory?.driver_warning ?? null; + const driverWarning = + rawDriverWarning?.driver_version && + rawDriverWarning.driver_version !== dismissedDriverWarning + ? rawDriverWarning + : null; // A CPU-only wheel is fixed by reinstalling torch, a dead runtime by the driver. const gpuMismatchMessage = gpuMismatch ? t( @@ -650,6 +672,48 @@ export function ResourcesTab() { + {driverWarning?.driver_version ? ( +
+

+ {t("settings.resources.gpu.driverIdleEvict", { + version: driverWarning.driver_version, + })}{" "} + {driverWarning.link?.startsWith("https://") ? ( + { + if (openLink(event.currentTarget.href)) + event.preventDefault(); + }} + className="inline-flex items-center gap-0.5 font-medium underline underline-offset-2" + > + {t("settings.resources.gpu.driverIdleEvictDetails")} + + + ) : null} +

+ +
+ ) : null} {/* Physically present, torch-unusable cards. Their own block, above the device rows and visually separate from them, because nothing here is selectable: the rows below are what a model can be loaded onto. */} diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts index bfb421f0f0..bc5099ff33 100644 --- a/studio/frontend/src/i18n/locales/ar.ts +++ b/studio/frontend/src/i18n/locales/ar.ts @@ -2113,6 +2113,10 @@ export const ar = { "PyTorch إصدار للـ CPU فقط ({version})، لذا لا يمكن استخدام وحدات GPU أدناه. أصلح التثبيت لاستعادة دعم GPU.", mismatchUnavailable: "لا يستطيع PyTorch ({version}) تهيئة وحدات GPU أدناه، لذا لا يمكن استخدامها. تحقق من تعريف كرت الشاشة أو أصلح التثبيت.", + driverIdleEvict: + "يحتوي برنامج تشغيل AMD {version} على خلل معروف قد يؤدي إلى تجمّد Windows عندما تكون وحدة GPU من AMD خاملة، وغالبًا مع وجود أكثر من GPU. حدّث إلى Adrenalin 26.9.2 أو أحدث.", + driverIdleEvictDetails: "التفاصيل", + dismissNotice: "إغلاق", unusableDevice: "غير قابل للاستخدام", unknownDevice: "GPU غير معروف", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts index 7f5f511ee4..0739b989f2 100644 --- a/studio/frontend/src/i18n/locales/de.ts +++ b/studio/frontend/src/i18n/locales/de.ts @@ -2142,6 +2142,10 @@ export const de = { "PyTorch ist ein reiner CPU-Build ({version}), daher können die GPUs unten nicht genutzt werden. Reparieren Sie die Installation, um die GPU-Unterstützung wiederherzustellen.", mismatchUnavailable: "PyTorch ({version}) kann die GPUs unten nicht initialisieren, daher sind sie nicht nutzbar. Prüfen Sie den Grafiktreiber oder reparieren Sie die Installation.", + driverIdleEvict: + "Der AMD-Treiber {version} hat einen bekannten Fehler, der Windows einfrieren kann, wenn eine AMD-GPU im Leerlauf ist, meist mit mehr als einer GPU. Aktualisieren Sie auf Adrenalin 26.9.2 oder neuer.", + driverIdleEvictDetails: "Details", + dismissNotice: "Ausblenden", unusableDevice: "nicht nutzbar", unknownDevice: "Unbekannte GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index 1ee30514a8..9b7d663429 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -2153,6 +2153,10 @@ export const en = { "PyTorch is a CPU-only build ({version}), so the GPUs below cannot be used. Repair the installation to restore GPU support.", mismatchUnavailable: "PyTorch ({version}) cannot initialise the GPUs below, so they cannot be used. Check the GPU driver, or repair the installation.", + driverIdleEvict: + "AMD driver {version} has a known bug that can freeze Windows when an AMD GPU sits idle, most often with more than one GPU. Update to Adrenalin 26.9.2 or later.", + driverIdleEvictDetails: "Details", + dismissNotice: "Dismiss", unusableDevice: "unusable", unknownDevice: "Unknown GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts index c13c5a50a8..e341211d01 100644 --- a/studio/frontend/src/i18n/locales/es.ts +++ b/studio/frontend/src/i18n/locales/es.ts @@ -2137,6 +2137,10 @@ export const es = { "PyTorch es una compilación solo para CPU ({version}), por lo que las GPU de abajo no se pueden usar. Repare la instalación para restaurar la compatibilidad con GPU.", mismatchUnavailable: "PyTorch ({version}) no puede inicializar las GPU de abajo, por lo que no se pueden usar. Revise el controlador de la GPU o repare la instalación.", + driverIdleEvict: + "El controlador de AMD {version} tiene un error conocido que puede congelar Windows cuando una GPU de AMD está inactiva, sobre todo con más de una GPU. Actualiza a Adrenalin 26.9.2 o posterior.", + driverIdleEvictDetails: "Detalles", + dismissNotice: "Descartar", unusableDevice: "no utilizable", unknownDevice: "GPU desconocida", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts index 58a4333914..0fdc1b3fb1 100644 --- a/studio/frontend/src/i18n/locales/fr.ts +++ b/studio/frontend/src/i18n/locales/fr.ts @@ -2141,6 +2141,10 @@ export const fr = { "PyTorch est une version CPU uniquement ({version}), les GPU ci-dessous ne peuvent donc pas être utilisés. Réparez l'installation pour rétablir la prise en charge du GPU.", mismatchUnavailable: "PyTorch ({version}) ne parvient pas à initialiser les GPU ci-dessous, ils ne peuvent donc pas être utilisés. Vérifiez le pilote graphique ou réparez l'installation.", + driverIdleEvict: + "Le pilote AMD {version} présente un bug connu qui peut figer Windows lorsqu'un GPU AMD reste inactif, le plus souvent avec plusieurs GPU. Mettez à jour vers Adrenalin 26.9.2 ou une version ultérieure.", + driverIdleEvictDetails: "Détails", + dismissNotice: "Ignorer", unusableDevice: "inutilisable", unknownDevice: "GPU inconnu", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.ts index 2acb976bdf..f5e73249a6 100644 --- a/studio/frontend/src/i18n/locales/he.ts +++ b/studio/frontend/src/i18n/locales/he.ts @@ -2194,6 +2194,10 @@ export const he = { "גרסת PyTorch מותאמת ל-CPU בלבד ({version}), ולכן לא ניתן להשתמש ב-GPUs שלמטה. תקן את ההתקנה כדי לשחזר תמיכה ב-GPU.", mismatchUnavailable: "גרסת PyTorch ({version}) אינה יכולה לאתחל את ה-GPUs שלמטה, ולכן לא ניתן להשתמש בהם. בדוק את מנהל ההתקן (Driver), או תקן את ההתקנה.", + driverIdleEvict: + "במנהל ההתקן של AMD {version} יש באג ידוע שעלול להקפיא את Windows כשמעבד גרפי של AMD במצב סרק, בעיקר כשיש יותר ממעבד גרפי אחד. עדכנו ל-Adrenalin 26.9.2 ומעלה.", + driverIdleEvictDetails: "פרטים", + dismissNotice: "סגירה", unusableDevice: "אינו שמיש", unknownDevice: "GPU לא ידוע", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts index 7203e8bb96..52a0d3a518 100644 --- a/studio/frontend/src/i18n/locales/hi.ts +++ b/studio/frontend/src/i18n/locales/hi.ts @@ -2116,6 +2116,10 @@ export const hi = { "PyTorch केवल-CPU बिल्ड ({version}) है, इसलिए नीचे दिए गए GPU उपयोग नहीं किए जा सकते। GPU समर्थन बहाल करने के लिए इंस्टॉलेशन की मरम्मत करें।", mismatchUnavailable: "PyTorch ({version}) नीचे दिए गए GPU को आरंभ नहीं कर पा रहा है, इसलिए उनका उपयोग नहीं किया जा सकता। GPU ड्राइवर जाँचें या इंस्टॉलेशन की मरम्मत करें।", + driverIdleEvict: + "AMD ड्राइवर {version} में एक ज्ञात बग है जिससे AMD GPU के निष्क्रिय रहने पर Windows फ़्रीज़ हो सकता है, ज़्यादातर एक से अधिक GPU होने पर। Adrenalin 26.9.2 या नए संस्करण में अपडेट करें।", + driverIdleEvictDetails: "विवरण", + dismissNotice: "खारिज करें", unusableDevice: "अनुपयोगी", unknownDevice: "अज्ञात GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.ts index eb8400cf9a..f67384867a 100644 --- a/studio/frontend/src/i18n/locales/it.ts +++ b/studio/frontend/src/i18n/locales/it.ts @@ -2101,6 +2101,10 @@ export const it = { "PyTorch è una build solo CPU ({version}), quindi le GPU sottostanti non possono essere usate. Ripara l'installazione per ripristinare il supporto GPU.", mismatchUnavailable: "PyTorch ({version}) non riesce a inizializzare le GPU sottostanti, quindi non possono essere usate. Controlla il driver della GPU o ripara l'installazione.", + driverIdleEvict: + "Il driver AMD {version} ha un bug noto che può bloccare Windows quando una GPU AMD resta inattiva, soprattutto con più di una GPU. Aggiorna ad Adrenalin 26.9.2 o successivo.", + driverIdleEvictDetails: "Dettagli", + dismissNotice: "Ignora", unusableDevice: "non utilizzabile", unknownDevice: "GPU sconosciuta", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index cdd86ca99d..e852d7529d 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -2093,6 +2093,10 @@ export const ja = { "PyTorch は CPU 専用ビルド ({version}) のため、下の GPU は利用できません。インストールを修復すると GPU サポートが復元されます。", mismatchUnavailable: "PyTorch ({version}) は下の GPU を初期化できないため、利用できません。GPU ドライバーを確認するか、インストールを修復してください。", + driverIdleEvict: + "AMD ドライバー {version} には、AMD GPU がアイドル状態のときに Windows がフリーズすることがある既知の不具合があります。特に GPU が複数ある環境で発生しやすくなります。Adrenalin 26.9.2 以降に更新してください。", + driverIdleEvictDetails: "詳細", + dismissNotice: "閉じる", unusableDevice: "利用不可", unknownDevice: "不明な GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts index 813510d1c2..2d670abeac 100644 --- a/studio/frontend/src/i18n/locales/ko.ts +++ b/studio/frontend/src/i18n/locales/ko.ts @@ -2109,6 +2109,10 @@ export const ko = { "PyTorch가 CPU 전용 빌드({version})이므로 아래 GPU를 사용할 수 없습니다. 설치를 복구하면 GPU 지원이 복원됩니다.", mismatchUnavailable: "PyTorch({version})가 아래 GPU를 초기화하지 못해 사용할 수 없습니다. GPU 드라이버를 확인하거나 설치를 복구하세요.", + driverIdleEvict: + "AMD 드라이버 {version}에는 AMD GPU가 유휴 상태일 때 Windows가 멈출 수 있는 알려진 버그가 있으며, GPU가 두 개 이상일 때 주로 발생합니다. Adrenalin 26.9.2 이상으로 업데이트하세요.", + driverIdleEvictDetails: "자세히", + dismissNotice: "닫기", unusableDevice: "사용 불가", unknownDevice: "알 수 없는 GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index db2c175a4d..4e9c1919b0 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -2125,6 +2125,10 @@ export const ptBR = { "O PyTorch é uma compilação somente CPU ({version}), então as GPUs abaixo não podem ser usadas. Repare a instalação para restaurar o suporte a GPU.", mismatchUnavailable: "O PyTorch ({version}) não consegue inicializar as GPUs abaixo, então elas não podem ser usadas. Verifique o driver da GPU ou repare a instalação.", + driverIdleEvict: + "O driver AMD {version} tem um bug conhecido que pode congelar o Windows quando uma GPU AMD fica ociosa, principalmente com mais de uma GPU. Atualize para o Adrenalin 26.9.2 ou posterior.", + driverIdleEvictDetails: "Detalhes", + dismissNotice: "Dispensar", unusableDevice: "inutilizável", unknownDevice: "GPU desconhecida", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts index 2f2d1f4f56..eb03f048db 100644 --- a/studio/frontend/src/i18n/locales/ru.ts +++ b/studio/frontend/src/i18n/locales/ru.ts @@ -2120,6 +2120,10 @@ export const ru = { "PyTorch собран только для CPU ({version}), поэтому GPU ниже использовать нельзя. Восстановите установку, чтобы вернуть поддержку GPU.", mismatchUnavailable: "PyTorch ({version}) не может инициализировать GPU ниже, поэтому использовать их нельзя. Проверьте драйвер видеокарты или восстановите установку.", + driverIdleEvict: + "В драйвере AMD {version} есть известная ошибка, из-за которой Windows может зависнуть, когда GPU AMD простаивает, чаще всего при нескольких GPU. Обновитесь до Adrenalin 26.9.2 или новее.", + driverIdleEvictDetails: "Подробнее", + dismissNotice: "Скрыть", unusableDevice: "недоступен", unknownDevice: "Неизвестный GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.ts index d7a3cefa3a..1631b36ef3 100644 --- a/studio/frontend/src/i18n/locales/sv.ts +++ b/studio/frontend/src/i18n/locales/sv.ts @@ -2266,6 +2266,10 @@ export const sv = { "PyTorch är en version endast för CPU ({version}), så GPU:erna nedan kan inte användas. Reparera installationen för att återställa GPU-stödet.", mismatchUnavailable: "PyTorch ({version}) kan inte initiera GPU:erna nedan, så de kan inte användas. Kontrollera GPU-drivrutinen eller reparera installationen.", + driverIdleEvict: + "AMD-drivrutinen {version} har en känd bugg som kan frysa Windows när ett AMD-grafikkort står inaktivt, oftast med fler än ett grafikkort. Uppdatera till Adrenalin 26.9.2 eller senare.", + driverIdleEvictDetails: "Detaljer", + dismissNotice: "Stäng", unusableDevice: "oanvändbar", unknownDevice: "Okänd GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index 495bef07c8..908d4a9c0a 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -2084,6 +2084,10 @@ export const zhCN = { noUsableGpu: "本机没有 PyTorch 可用的 GPU。", mismatchCpuBuild: "PyTorch 是仅 CPU 版本（{version}），因此无法使用下方的 GPU。修复安装即可恢复 GPU 支持。", mismatchUnavailable: "PyTorch（{version}）无法初始化下方的 GPU，因此无法使用。请检查显卡驱动，或修复安装。", + driverIdleEvict: + "AMD 驱动 {version} 存在已知缺陷，AMD GPU 空闲时可能导致 Windows 卡死，多 GPU 环境下最常见。请更新到 Adrenalin 26.9.2 或更高版本。", + driverIdleEvictDetails: "详情", + dismissNotice: "关闭", unusableDevice: "不可用", unknownDevice: "未知 GPU", deviceWithIndex: "GPU {index}", diff --git a/studio/frontend/tests/amd-driver-idle-evict-notice.test.ts b/studio/frontend/tests/amd-driver-idle-evict-notice.test.ts new file mode 100644 index 0000000000..0c933cb132 --- /dev/null +++ b/studio/frontend/tests/amd-driver-idle-evict-notice.test.ts @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// The System tab's notice for a Windows AMD driver with the idle-eviction bug +// (ROCm/TheRock#7221), from /api/system's gpu.driver_warning. Lifted by regex like +// gpu-torch-mismatch.test.ts, since the tab pulls in the whole app. + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { readSrcAsync, registerBundlerResolver } from "./helpers/kit.ts"; + +registerBundlerResolver(); + +const { en } = await import("../src/i18n/locales/en.ts"); +const tabSrc = await readSrcAsync("features/settings/tabs/resources-tab.tsx"); + +function lift(pattern: RegExp, what: string): string { + const found = pattern.exec(tabSrc); + assert.ok(found, `could not find ${what} in resources-tab.tsx`); + return found[0]; +} + +const derivation = [ + lift(/const rawDriverWarning = [\s\S]*?;/, "rawDriverWarning"), + lift(/const driverWarning =[\s\S]*?;/, "driverWarning"), +].join("\n"); + +interface Warning { + driver_version?: string; + severity?: string; + link?: string; +} + +function noticeFor( + gpuInventory: { driver_warning?: Warning | null } | null, + dismissedDriverWarning: string | null = null, +) { + const run = new Function( + "gpuInventory", + "dismissedDriverWarning", + `${derivation}\n return driverWarning;`, + ); + return run(gpuInventory, dismissedDriverWarning) as Warning | null; +} + +const BROKEN = { + driver_version: "32.0.31041.1004", + severity: "warning", + link: "https://github.com/ROCm/TheRock/issues/7221", +}; + +test("a flagged driver shows the notice", () => { + assert.deepEqual(noticeFor({ driver_warning: BROKEN }), BROKEN); +}); + +test("no warning, or a host not read yet, shows nothing", () => { + for (const inventory of [ + null, + {}, + { driver_warning: null }, + { driver_warning: {} }, + ]) { + assert.equal(noticeFor(inventory), null); + } +}); + +test("dismissing hides that driver only, so a different bad driver shows again", () => { + assert.equal(noticeFor({ driver_warning: BROKEN }, "32.0.31041.1004"), null); + const other = { ...BROKEN, driver_version: "32.0.31014.1001" }; + assert.deepEqual( + noticeFor({ driver_warning: other }, "32.0.31041.1004"), + other, + ); +}); + +test("the message names the version and the fixed release", () => { + const message = en.settings.resources.gpu.driverIdleEvict; + assert.match(message, /\{version\}/); + assert.match(message, /Adrenalin 26\.9\.2/); +}); + +test("only an https link is rendered", () => { + assert.match(tabSrc, /driverWarning\.link\?\.startsWith\("https:\/\/"\)/); +}); diff --git a/studio/frontend/tests/settings-panel-prefs.test.ts b/studio/frontend/tests/settings-panel-prefs.test.ts index 918fc14b58..99c9ccf1dc 100644 --- a/studio/frontend/tests/settings-panel-prefs.test.ts +++ b/studio/frontend/tests/settings-panel-prefs.test.ts @@ -222,6 +222,21 @@ test("an unreadable record leaves the defaults", () => { assert.equal(out.fineTuneAction, "train"); }); +test("a dismissed driver notice round-trips, and a non-string record is dropped", () => { + useSettingsPanelPrefsStore.getState().dismissDriverWarning("32.0.31041.1004"); + assert.equal( + JSON.parse(store.get(KEY) as string).state.dismissedDriverWarning, + "32.0.31041.1004", + ); + const merged = useSettingsPanelPrefsStore.persist.getOptions().merge; + assert.ok(merged); + const state = useSettingsPanelPrefsStore.getState(); + assert.equal( + merged({ dismissedDriverWarning: 42 }, state).dismissedDriverWarning, + null, + ); +}); + // Last: it rehydrates the store. Corrupt JSON must not take settings down. test("corrupt JSON does not break the store", async () => { store.set(KEY, "{not json");