diff --git a/tests/config/test_speculative_draft_hf_overrides.py b/tests/config/test_speculative_draft_hf_overrides.py new file mode 100644 index 0000000000..ddb8752a80 --- /dev/null +++ b/tests/config/test_speculative_draft_hf_overrides.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for SpeculativeConfig.compose_draft_hf_overrides. + +Callable ``hf_overrides`` on the target model config (e.g. the +``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``) +must also be applied when building the draft ``ModelConfig``. Otherwise a +draft belonging to a large target model is instantiated at full size even +when the target itself is shrunk — which is what kept spec-decode archs like +``EagleMistralLarge3ForCausalLM`` stuck at ``is_available_online=False`` +("TODO: revert once figuring out OOM in CI"). +""" + +import functools + +import pytest +from transformers import PretrainedConfig + +from vllm.config.speculative import SpeculativeConfig + + +def _make_hf_config(**kwargs) -> PretrainedConfig: + defaults = dict( + architectures=["LlamaForCausalLM"], + model_type="llama", + num_hidden_layers=64, + ) + defaults.update(kwargs) + return PretrainedConfig(**defaults) + + +@pytest.mark.cpu_test +def test_dict_overrides_are_not_forwarded_to_draft(): + """Dict overrides are target-specific key patches; the draft must get + only the architecture-mapping override.""" + composed = SpeculativeConfig.compose_draft_hf_overrides( + {"max_position_embeddings": 1234} + ) + assert composed is SpeculativeConfig.hf_config_override + + +@pytest.mark.cpu_test +def test_none_overrides_fall_back_to_arch_mapping(): + composed = SpeculativeConfig.compose_draft_hf_overrides(None) + assert composed is SpeculativeConfig.hf_config_override + + +@pytest.mark.cpu_test +def test_callable_overrides_reach_the_draft_config(): + """A callable override (config-to-config transform) composes with the + architecture-mapping override and is applied to the draft config.""" + + def shrink(hf_config: PretrainedConfig) -> PretrainedConfig: + hf_config.num_hidden_layers = 1 + return hf_config + + composed = SpeculativeConfig.compose_draft_hf_overrides(shrink) + assert composed is not SpeculativeConfig.hf_config_override + + out = composed(_make_hf_config()) + # The shrink transform must have been applied to the draft config. + assert out.num_hidden_layers == 1 + + +@pytest.mark.cpu_test +def test_arch_mapping_applies_before_callable_override(): + """The static arch-mapping override runs first, so the user callable + observes (and may adjust) the post-mapping config.""" + seen_architectures: list[str] = [] + + def record(hf_config: PretrainedConfig) -> PretrainedConfig: + seen_architectures.append(hf_config.architectures[0]) + return hf_config + + composed = SpeculativeConfig.compose_draft_hf_overrides(record) + + # MiMo is one of the arch-mapped model types: hf_config_override + # rewrites architectures to ["MiMoMTPModel"]. + mimo = _make_hf_config( + architectures=["MiMoForCausalLM"], + model_type="mimo", + num_nextn_predict_layers=1, + ) + composed(mimo) + assert seen_architectures == ["MiMoMTPModel"] + + +def _module_level_shrink(hf_config: PretrainedConfig) -> PretrainedConfig: + hf_config.num_hidden_layers = 1 + return hf_config + + +@pytest.mark.cpu_test +def test_composed_override_is_picklable(): + """The draft ``ModelConfig`` is sent to spawned engine-core processes, so + the composed override must be picklable. A nested local closure is not + (it raised ``Can't get local object`` on DFlashDraftModel); a + ``functools.partial`` over a module-referenceable static method is. + Guard against regressing to a closure.""" + composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink) + + assert isinstance(composed, functools.partial) + assert composed.func is SpeculativeConfig._apply_composed_hf_override + + out = composed(_make_hf_config()) + assert out.num_hidden_layers == 1 diff --git a/tests/models/registry.py b/tests/models/registry.py index 2e9f05ec8b..b5861b4a10 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1492,8 +1492,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "EagleMistralLarge3ForCausalLM": _HfExamplesInfo( "mistralai/Mistral-Large-3-675B-Instruct-2512", speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle", - # TODO: revert once figuring out OOM in CI - is_available_online=False, ), "LlamaForCausalLMEagle3": _HfExamplesInfo( "Qwen/Qwen3-8B", diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 7fc4700fa2..2e0126368c 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy +import functools +from collections.abc import Callable from typing import TYPE_CHECKING, Any, Literal, get_args from pydantic import Field, SkipValidation, field_validator, model_validator @@ -588,6 +590,40 @@ class SpeculativeConfig: return hf_config + @staticmethod + def _apply_composed_hf_override( + target_hf_overrides: Callable[[PretrainedConfig], PretrainedConfig], + hf_config: PretrainedConfig, + ) -> PretrainedConfig: + hf_config = SpeculativeConfig.hf_config_override(hf_config) + return target_hf_overrides(hf_config) + + @staticmethod + def compose_draft_hf_overrides( + target_hf_overrides: HfOverrides | None, + ) -> Callable[[PretrainedConfig], PretrainedConfig]: + """Build the ``hf_overrides`` for the draft ``ModelConfig``. + + Callable overrides on the target are config-to-config transforms + (e.g. test harnesses shrinking ``num_hidden_layers``) and must also + reach the draft config — otherwise a draft belonging to a large + target is instantiated at full size even when the target is shrunk. + Dict overrides are target-specific key patches and are not applied + to the draft. + + The composed override must stay picklable: the draft ``ModelConfig`` + is sent to spawned engine-core processes, so a local closure would + fail with ``Can't get local object`` during pickling. Bind the + target via ``functools.partial`` over a module-referenceable static + method instead. + """ + if not callable(target_hf_overrides): + return SpeculativeConfig.hf_config_override + + return functools.partial( + SpeculativeConfig._apply_composed_hf_override, target_hf_overrides + ) + def __post_init__(self): # Note: "method" is a new parameter that helps to extend the # configuration of non-model-based proposers, and the "model" parameter @@ -752,7 +788,12 @@ class SpeculativeConfig: if self.method == "medusa": draft_hf_overrides = {"model_type": "medusa"} else: - draft_hf_overrides = SpeculativeConfig.hf_config_override + # Compose any callable hf_overrides set on the target so the + # draft config receives the same transform (e.g. the test + # shrink). Dict overrides stay target-only. + draft_hf_overrides = SpeculativeConfig.compose_draft_hf_overrides( + self.target_model_config.hf_overrides + ) self.draft_model_config = ModelConfig( model=self.model, runner="draft",