[Bugfix] Forward callable hf_overrides to the draft model config (#45352)

Signed-off-by: HumphreySun98 <humphreysun98@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Humphrey
2026-07-06 21:12:38 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Cyrus Leung
parent 2f71b2bd9f
commit cbe9c40f99
3 changed files with 148 additions and 3 deletions
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for SpeculativeConfig.compose_draft_hf_overrides.
Callable ``hf_overrides`` on the target model config (e.g. the
``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``)
must also be applied when building the draft ``ModelConfig``. Otherwise a
draft belonging to a large target model is instantiated at full size even
when the target itself is shrunk — which is what kept spec-decode archs like
``EagleMistralLarge3ForCausalLM`` stuck at ``is_available_online=False``
("TODO: revert once figuring out OOM in CI").
"""
import functools
import pytest
from transformers import PretrainedConfig
from vllm.config.speculative import SpeculativeConfig
def _make_hf_config(**kwargs) -> PretrainedConfig:
defaults = dict(
architectures=["LlamaForCausalLM"],
model_type="llama",
num_hidden_layers=64,
)
defaults.update(kwargs)
return PretrainedConfig(**defaults)
@pytest.mark.cpu_test
def test_dict_overrides_are_not_forwarded_to_draft():
"""Dict overrides are target-specific key patches; the draft must get
only the architecture-mapping override."""
composed = SpeculativeConfig.compose_draft_hf_overrides(
{"max_position_embeddings": 1234}
)
assert composed is SpeculativeConfig.hf_config_override
@pytest.mark.cpu_test
def test_none_overrides_fall_back_to_arch_mapping():
composed = SpeculativeConfig.compose_draft_hf_overrides(None)
assert composed is SpeculativeConfig.hf_config_override
@pytest.mark.cpu_test
def test_callable_overrides_reach_the_draft_config():
"""A callable override (config-to-config transform) composes with the
architecture-mapping override and is applied to the draft config."""
def shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
hf_config.num_hidden_layers = 1
return hf_config
composed = SpeculativeConfig.compose_draft_hf_overrides(shrink)
assert composed is not SpeculativeConfig.hf_config_override
out = composed(_make_hf_config())
# The shrink transform must have been applied to the draft config.
assert out.num_hidden_layers == 1
@pytest.mark.cpu_test
def test_arch_mapping_applies_before_callable_override():
"""The static arch-mapping override runs first, so the user callable
observes (and may adjust) the post-mapping config."""
seen_architectures: list[str] = []
def record(hf_config: PretrainedConfig) -> PretrainedConfig:
seen_architectures.append(hf_config.architectures[0])
return hf_config
composed = SpeculativeConfig.compose_draft_hf_overrides(record)
# MiMo is one of the arch-mapped model types: hf_config_override
# rewrites architectures to ["MiMoMTPModel"].
mimo = _make_hf_config(
architectures=["MiMoForCausalLM"],
model_type="mimo",
num_nextn_predict_layers=1,
)
composed(mimo)
assert seen_architectures == ["MiMoMTPModel"]
def _module_level_shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
hf_config.num_hidden_layers = 1
return hf_config
@pytest.mark.cpu_test
def test_composed_override_is_picklable():
"""The draft ``ModelConfig`` is sent to spawned engine-core processes, so
the composed override must be picklable. A nested local closure is not
(it raised ``Can't get local object`` on DFlashDraftModel); a
``functools.partial`` over a module-referenceable static method is.
Guard against regressing to a closure."""
composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink)
assert isinstance(composed, functools.partial)
assert composed.func is SpeculativeConfig._apply_composed_hf_override
out = composed(_make_hf_config())
assert out.num_hidden_layers == 1
-2
View File
@@ -1492,8 +1492,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
"mistralai/Mistral-Large-3-675B-Instruct-2512",
speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
# TODO: revert once figuring out OOM in CI
is_available_online=False,
),
"LlamaForCausalLMEagle3": _HfExamplesInfo(
"Qwen/Qwen3-8B",
+42 -1
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import functools
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Literal, get_args
from pydantic import Field, SkipValidation, field_validator, model_validator
@@ -588,6 +590,40 @@ class SpeculativeConfig:
return hf_config
@staticmethod
def _apply_composed_hf_override(
target_hf_overrides: Callable[[PretrainedConfig], PretrainedConfig],
hf_config: PretrainedConfig,
) -> PretrainedConfig:
hf_config = SpeculativeConfig.hf_config_override(hf_config)
return target_hf_overrides(hf_config)
@staticmethod
def compose_draft_hf_overrides(
target_hf_overrides: HfOverrides | None,
) -> Callable[[PretrainedConfig], PretrainedConfig]:
"""Build the ``hf_overrides`` for the draft ``ModelConfig``.
Callable overrides on the target are config-to-config transforms
(e.g. test harnesses shrinking ``num_hidden_layers``) and must also
reach the draft config — otherwise a draft belonging to a large
target is instantiated at full size even when the target is shrunk.
Dict overrides are target-specific key patches and are not applied
to the draft.
The composed override must stay picklable: the draft ``ModelConfig``
is sent to spawned engine-core processes, so a local closure would
fail with ``Can't get local object`` during pickling. Bind the
target via ``functools.partial`` over a module-referenceable static
method instead.
"""
if not callable(target_hf_overrides):
return SpeculativeConfig.hf_config_override
return functools.partial(
SpeculativeConfig._apply_composed_hf_override, target_hf_overrides
)
def __post_init__(self):
# Note: "method" is a new parameter that helps to extend the
# configuration of non-model-based proposers, and the "model" parameter
@@ -752,7 +788,12 @@ class SpeculativeConfig:
if self.method == "medusa":
draft_hf_overrides = {"model_type": "medusa"}
else:
draft_hf_overrides = SpeculativeConfig.hf_config_override
# Compose any callable hf_overrides set on the target so the
# draft config receives the same transform (e.g. the test
# shrink). Dict overrides stay target-only.
draft_hf_overrides = SpeculativeConfig.compose_draft_hf_overrides(
self.target_model_config.hf_overrides
)
self.draft_model_config = ModelConfig(
model=self.model,
runner="draft",