mirror of
https://github.com/unslothai/unsloth.git
synced 2026-10-21 14:00:32 +00:00
50b9deacb0
Comment and docstring changes only, in test files, split out of #12968 so the test-only part can merge first.
366 lines
13 KiB
Python
366 lines
13 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
"""Fails on any temporary patch that raised; declining cleanly is a normal return, not a failure."""
|
|
|
|
import ast
|
|
import gc
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import weakref
|
|
|
|
import pytest
|
|
|
|
_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
_UTILS = _ROOT / "unsloth" / "models" / "_utils.py"
|
|
|
|
_BEGIN = "UNSLOTH_PATCH_REPORT_BEGIN"
|
|
_END = "UNSLOTH_PATCH_REPORT_END"
|
|
|
|
_CHILD = """
|
|
import json
|
|
import unsloth # noqa: F401
|
|
import unsloth.models._utils as _utils
|
|
|
|
# `import unsloth` only runs the "init" pass. The other two run from
|
|
# unsloth_compile_transformers around the compiler, and a phase-aware patch
|
|
# branches on which one it is (patch_Gemma4_static_cache_backport and
|
|
# patch_gpt_oss_for_grpo both return immediately unless phase == post_compile),
|
|
# so a patch can be perfectly quiet at import and raise only in a compile pass.
|
|
# Running the two passes here executes those branches without standing up a
|
|
# model or a compiler. It is not the full production state -- post_compile
|
|
# normally runs after _unsloth_compile_transformers has swapped classes -- so
|
|
# this catches a patch that raises on its own terms, not one that only raises
|
|
# against a compiled model.
|
|
_utils._run_temporary_patches("pre_compile")
|
|
_utils._run_temporary_patches("post_compile")
|
|
|
|
{injection}
|
|
|
|
report = {{}}
|
|
for phase, outcome in _utils.TEMPORARY_PATCH_OUTCOMES.items():
|
|
report[phase] = {{
|
|
"completed": [getattr(p, "__name__", repr(p)) for p in outcome["completed"]],
|
|
"raised": [
|
|
[getattr(p, "__name__", repr(p)), type(e).__name__, str(e)]
|
|
for p, e in outcome["raised"]
|
|
],
|
|
}}
|
|
print({begin!r})
|
|
print(json.dumps(report))
|
|
print({end!r})
|
|
"""
|
|
|
|
_NO_ACCELERATOR = (
|
|
"Unsloth cannot find any torch accelerator",
|
|
"No CUDA GPUs are available",
|
|
"Torch not compiled with CUDA enabled",
|
|
)
|
|
|
|
|
|
def _spawn(code, **extra_env):
|
|
path = [str(_ROOT)]
|
|
if os.environ.get("PYTHONPATH"):
|
|
path.append(os.environ["PYTHONPATH"])
|
|
return subprocess.run(
|
|
[sys.executable, "-c", code],
|
|
capture_output = True,
|
|
text = True,
|
|
env = dict(os.environ, PYTHONPATH = os.pathsep.join(path), **extra_env),
|
|
timeout = 1800,
|
|
)
|
|
|
|
|
|
def _collect(injection = ""):
|
|
"""Retries with UNSLOTH_ALLOW_CPU=1 only on the no-accelerator symptom, so no host or OS is probed."""
|
|
code = _CHILD.format(injection = injection, begin = _BEGIN, end = _END)
|
|
result = _spawn(code)
|
|
if result.returncode != 0 and any(m in result.stderr for m in _NO_ACCELERATOR):
|
|
result = _spawn(code, UNSLOTH_ALLOW_CPU = "1")
|
|
if result.returncode != 0 or _BEGIN not in result.stdout:
|
|
raise AssertionError(
|
|
"could not import unsloth to read the temporary patch outcomes\n"
|
|
f"exit {result.returncode}\n--- stdout ---\n{result.stdout}\n"
|
|
f"--- stderr ---\n{result.stderr}"
|
|
)
|
|
body = result.stdout.split(_BEGIN, 1)[1].split(_END, 1)[0]
|
|
return json.loads(body)
|
|
|
|
|
|
def _assert_no_patch_raised(report):
|
|
"""The gate. Fails naming every patch that raised, and with what."""
|
|
failures = []
|
|
for phase in sorted(report):
|
|
for name, exception_type, message in report[phase]["raised"]:
|
|
failures.append(f" {name} (phase {phase}): {exception_type}: {message}")
|
|
assert not failures, (
|
|
"PATCH COVERAGE LOST: these temporary patches raised and were swallowed by the "
|
|
"fail-soft handler in _run_temporary_patches, so they are no longer applied. A "
|
|
"patch that no longer has anything to patch on this library version must return "
|
|
"cleanly, not raise.\n" + "\n".join(failures)
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope = "module")
|
|
def _clean_report():
|
|
return _collect()
|
|
|
|
|
|
_PHASES = ("init", "pre_compile", "post_compile")
|
|
|
|
|
|
def test_no_temporary_patch_raised_on_this_library_version(_clean_report):
|
|
assert _clean_report, "no phase was recorded at all, the bookkeeping is not running"
|
|
missing = [phase for phase in _PHASES if phase not in _clean_report]
|
|
assert not missing, f"no outcome was recorded for {missing}"
|
|
_assert_no_patch_raised(_clean_report)
|
|
|
|
|
|
@pytest.mark.parametrize("phase", _PHASES)
|
|
def test_every_pass_actually_ran_some_patches(_clean_report, phase):
|
|
# Guards the degenerate green of an empty TEMPORARY_PATCHES or an undriven pass.
|
|
assert _clean_report[phase][
|
|
"completed"
|
|
], f"no temporary patch completed in the {phase} pass, so the gate above is vacuous there"
|
|
|
|
|
|
# Controls: gate must catch a breaking patch but ignore one that declines on this version.
|
|
|
|
|
|
_RAISES = """
|
|
def _mutation_control_patch_that_raises():
|
|
# What patch_merge_quantization_configs did in #3130: a SyntaxError out of
|
|
# an exec'd string, from inside a patch body.
|
|
raise SyntaxError("mutation control: this patch is broken")
|
|
|
|
_utils.TEMPORARY_PATCHES.append(_mutation_control_patch_that_raises)
|
|
_utils._run_temporary_patches("init")
|
|
"""
|
|
|
|
_DECLINES = """
|
|
def _mutation_control_patch_that_declines(phase):
|
|
# The shape of every version-conditional patch in unsloth_zoo: look for the
|
|
# thing to patch, and return without touching anything when this library
|
|
# version does not have it.
|
|
try:
|
|
import a_module_no_transformers_release_ships # noqa: F401
|
|
except ImportError:
|
|
return
|
|
raise AssertionError("unreachable")
|
|
|
|
_utils.TEMPORARY_PATCHES.append(_mutation_control_patch_that_declines)
|
|
_utils._run_temporary_patches("init")
|
|
"""
|
|
|
|
|
|
_RAISES_ONLY_AT_COMPILE_TIME = """
|
|
def _mutation_control_patch_quiet_until_compile(phase = "post_compile"):
|
|
# The shape the import-only gate could not see: the same phase test the two
|
|
# real phase-aware patches use, with a body that no longer works.
|
|
if phase != "post_compile": return
|
|
raise AttributeError("mutation control: this compile-time patch is broken")
|
|
|
|
_utils.TEMPORARY_PATCHES.append(_mutation_control_patch_quiet_until_compile)
|
|
_utils._run_temporary_patches("init")
|
|
_utils._run_temporary_patches("post_compile")
|
|
"""
|
|
|
|
|
|
def test_mutation_control_a_patch_that_only_breaks_at_compile_time_turns_the_gate_red():
|
|
report = _collect(injection = _RAISES_ONLY_AT_COMPILE_TIME)
|
|
assert (
|
|
"_mutation_control_patch_quiet_until_compile" in report["init"]["completed"]
|
|
), "the control has to be quiet during import, or it proves nothing about the compile passes"
|
|
with pytest.raises(AssertionError) as caught:
|
|
_assert_no_patch_raised(report)
|
|
message = str(caught.value)
|
|
assert "_mutation_control_patch_quiet_until_compile" in message, message
|
|
assert "phase post_compile" in message, message
|
|
|
|
|
|
def test_mutation_control_a_raising_patch_turns_the_gate_red():
|
|
report = _collect(injection = _RAISES)
|
|
with pytest.raises(AssertionError) as caught:
|
|
_assert_no_patch_raised(report)
|
|
message = str(caught.value)
|
|
assert "_mutation_control_patch_that_raises" in message, message
|
|
assert "SyntaxError" in message, message
|
|
|
|
|
|
def test_mutation_control_a_cleanly_declining_patch_does_not():
|
|
report = _collect(injection = _DECLINES)
|
|
_assert_no_patch_raised(report)
|
|
assert (
|
|
"_mutation_control_patch_that_declines" in report["init"]["completed"]
|
|
), "a patch that declined cleanly was not recorded as completed"
|
|
|
|
|
|
class _CollectingLogger:
|
|
def __init__(self):
|
|
self.warnings = []
|
|
|
|
def warning(self, message):
|
|
self.warnings.append(message)
|
|
|
|
|
|
def _isolated(patches, logger, outcomes):
|
|
source = _UTILS.read_text(encoding = "utf-8")
|
|
tree = ast.parse(source)
|
|
for node in tree.body:
|
|
if isinstance(node, ast.FunctionDef) and node.name == "_run_temporary_patches":
|
|
segment = ast.get_source_segment(source, node)
|
|
break
|
|
else:
|
|
raise AssertionError("_run_temporary_patches is gone from unsloth/models/_utils.py")
|
|
namespace = {
|
|
"TEMPORARY_PATCHES": patches,
|
|
"logger": logger,
|
|
"TEMPORARY_PATCH_OUTCOMES": outcomes,
|
|
}
|
|
exec(compile(segment, str(_UTILS), "exec"), namespace)
|
|
return namespace["_run_temporary_patches"]
|
|
|
|
|
|
def test_outcomes_separate_raising_from_returning():
|
|
def declines():
|
|
return None
|
|
|
|
def explodes():
|
|
raise RuntimeError("boom")
|
|
|
|
outcomes = {}
|
|
logger = _CollectingLogger()
|
|
_isolated([declines, explodes], logger, outcomes)("init")
|
|
|
|
assert [p.__name__ for p in outcomes["init"]["completed"]] == ["declines"]
|
|
assert [p.__name__ for p, _ in outcomes["init"]["raised"]] == ["explodes"]
|
|
assert len(logger.warnings) == 1, "fail-soft logging must be unchanged"
|
|
|
|
|
|
def test_outcomes_are_recorded_per_phase():
|
|
def declines(phase):
|
|
return None
|
|
|
|
outcomes = {}
|
|
run = _isolated([declines], _CollectingLogger(), outcomes)
|
|
run("init")
|
|
run("pre_compile")
|
|
|
|
assert sorted(outcomes) == ["init", "pre_compile"]
|
|
|
|
|
|
def test_a_repeated_pass_replaces_rather_than_grows_its_phase():
|
|
# `_run_temporary_patches` reruns on every model load, so appending records would leak.
|
|
def declines():
|
|
return None
|
|
|
|
outcomes = {}
|
|
run = _isolated([declines], _CollectingLogger(), outcomes)
|
|
for _ in range(5):
|
|
run("pre_compile")
|
|
|
|
assert len(outcomes["pre_compile"]["completed"]) == 1
|
|
|
|
|
|
def test_a_recorded_failure_does_not_pin_the_frames_it_raised_from():
|
|
# A raising patch stores an exception whose traceback pins frames; init entry lives forever.
|
|
class _Held:
|
|
pass
|
|
|
|
probes = []
|
|
|
|
def explodes():
|
|
heavy = _Held()
|
|
probes.append(weakref.ref(heavy))
|
|
raise RuntimeError("boom")
|
|
|
|
outcomes = {}
|
|
run = _isolated([explodes], _CollectingLogger(), outcomes)
|
|
run("init")
|
|
|
|
patch, exception = outcomes["init"]["raised"][0]
|
|
assert patch is explodes
|
|
assert isinstance(exception, RuntimeError), "the exception itself must still be recorded"
|
|
assert str(exception) == "boom", "the message the gate reports must survive the trim"
|
|
assert exception.__traceback__ is None, (
|
|
"the recorded exception still carries its traceback, which keeps the failed patch's "
|
|
"frames and their locals alive for the lifetime of the phase entry"
|
|
)
|
|
|
|
gc.collect()
|
|
assert probes[0]() is None, (
|
|
"an object local to the failed patch is still alive after the pass, so the record "
|
|
"is pinning the raising frame"
|
|
)
|
|
|
|
|
|
def test_an_exception_that_refuses_the_trim_does_not_break_the_pass():
|
|
# Traceback attrs can be property-shadowed; the trim runs in `import unsloth` so must not raise.
|
|
class _Stubborn(Exception):
|
|
@property
|
|
def __traceback__(self):
|
|
return None
|
|
|
|
@__traceback__.setter
|
|
def __traceback__(self, value):
|
|
raise ValueError("this exception will not give up its traceback")
|
|
|
|
def explodes():
|
|
raise _Stubborn("boom")
|
|
|
|
def runs_after():
|
|
return None
|
|
|
|
outcomes = {}
|
|
logger = _CollectingLogger()
|
|
_isolated([explodes, runs_after], logger, outcomes)("init")
|
|
|
|
assert [p.__name__ for p, _ in outcomes["init"]["raised"]] == ["explodes"]
|
|
assert [p.__name__ for p in outcomes["init"]["completed"]] == ["runs_after"]
|
|
assert len(logger.warnings) == 1
|
|
|
|
|
|
def test_an_unnamed_callable_does_not_break_the_recording():
|
|
class _Callable:
|
|
def __call__(self):
|
|
return None
|
|
|
|
outcomes = {}
|
|
logger = _CollectingLogger()
|
|
_isolated([_Callable()], logger, outcomes)("init")
|
|
|
|
assert len(outcomes["init"]["completed"]) == 1
|
|
assert logger.warnings == []
|
|
|
|
|
|
def test_the_recording_is_wired_into_the_patch_loop():
|
|
source = _UTILS.read_text(encoding = "utf-8")
|
|
tree = ast.parse(source)
|
|
for node in tree.body:
|
|
if isinstance(node, ast.FunctionDef) and node.name == "_run_temporary_patches":
|
|
segment = ast.get_source_segment(source, node)
|
|
break
|
|
else:
|
|
raise AssertionError("_run_temporary_patches is gone from unsloth/models/_utils.py")
|
|
|
|
assert "TEMPORARY_PATCH_OUTCOMES[phase]" in segment
|
|
assert "raised.append(" in segment
|
|
assert "completed.append(" in segment
|
|
assert (
|
|
"TEMPORARY_PATCH_OUTCOMES = {}" in source
|
|
), "the module level record the gate reads is gone"
|