[Bugfix] Forward callable hf_overrides to the draft model config (#45352)
Signed-off-by: HumphreySun98 <humphreysun98@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Cyrus Leung
parent
2f71b2bd9f
commit
cbe9c40f99
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for SpeculativeConfig.compose_draft_hf_overrides.
|
||||
|
||||
Callable ``hf_overrides`` on the target model config (e.g. the
|
||||
``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``)
|
||||
must also be applied when building the draft ``ModelConfig``. Otherwise a
|
||||
draft belonging to a large target model is instantiated at full size even
|
||||
when the target itself is shrunk — which is what kept spec-decode archs like
|
||||
``EagleMistralLarge3ForCausalLM`` stuck at ``is_available_online=False``
|
||||
("TODO: revert once figuring out OOM in CI").
|
||||
"""
|
||||
|
||||
import functools
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
|
||||
|
||||
def _make_hf_config(**kwargs) -> PretrainedConfig:
|
||||
defaults = dict(
|
||||
architectures=["LlamaForCausalLM"],
|
||||
model_type="llama",
|
||||
num_hidden_layers=64,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return PretrainedConfig(**defaults)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_dict_overrides_are_not_forwarded_to_draft():
|
||||
"""Dict overrides are target-specific key patches; the draft must get
|
||||
only the architecture-mapping override."""
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(
|
||||
{"max_position_embeddings": 1234}
|
||||
)
|
||||
assert composed is SpeculativeConfig.hf_config_override
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_none_overrides_fall_back_to_arch_mapping():
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(None)
|
||||
assert composed is SpeculativeConfig.hf_config_override
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_callable_overrides_reach_the_draft_config():
|
||||
"""A callable override (config-to-config transform) composes with the
|
||||
architecture-mapping override and is applied to the draft config."""
|
||||
|
||||
def shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
hf_config.num_hidden_layers = 1
|
||||
return hf_config
|
||||
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(shrink)
|
||||
assert composed is not SpeculativeConfig.hf_config_override
|
||||
|
||||
out = composed(_make_hf_config())
|
||||
# The shrink transform must have been applied to the draft config.
|
||||
assert out.num_hidden_layers == 1
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_arch_mapping_applies_before_callable_override():
|
||||
"""The static arch-mapping override runs first, so the user callable
|
||||
observes (and may adjust) the post-mapping config."""
|
||||
seen_architectures: list[str] = []
|
||||
|
||||
def record(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
seen_architectures.append(hf_config.architectures[0])
|
||||
return hf_config
|
||||
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(record)
|
||||
|
||||
# MiMo is one of the arch-mapped model types: hf_config_override
|
||||
# rewrites architectures to ["MiMoMTPModel"].
|
||||
mimo = _make_hf_config(
|
||||
architectures=["MiMoForCausalLM"],
|
||||
model_type="mimo",
|
||||
num_nextn_predict_layers=1,
|
||||
)
|
||||
composed(mimo)
|
||||
assert seen_architectures == ["MiMoMTPModel"]
|
||||
|
||||
|
||||
def _module_level_shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
hf_config.num_hidden_layers = 1
|
||||
return hf_config
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_composed_override_is_picklable():
|
||||
"""The draft ``ModelConfig`` is sent to spawned engine-core processes, so
|
||||
the composed override must be picklable. A nested local closure is not
|
||||
(it raised ``Can't get local object`` on DFlashDraftModel); a
|
||||
``functools.partial`` over a module-referenceable static method is.
|
||||
Guard against regressing to a closure."""
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink)
|
||||
|
||||
assert isinstance(composed, functools.partial)
|
||||
assert composed.func is SpeculativeConfig._apply_composed_hf_override
|
||||
|
||||
out = composed(_make_hf_config())
|
||||
assert out.num_hidden_layers == 1
|
||||
@@ -1492,8 +1492,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
|
||||
"mistralai/Mistral-Large-3-675B-Instruct-2512",
|
||||
speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
|
||||
# TODO: revert once figuring out OOM in CI
|
||||
is_available_online=False,
|
||||
),
|
||||
"LlamaForCausalLMEagle3": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Literal, get_args
|
||||
|
||||
from pydantic import Field, SkipValidation, field_validator, model_validator
|
||||
@@ -588,6 +590,40 @@ class SpeculativeConfig:
|
||||
|
||||
return hf_config
|
||||
|
||||
@staticmethod
|
||||
def _apply_composed_hf_override(
|
||||
target_hf_overrides: Callable[[PretrainedConfig], PretrainedConfig],
|
||||
hf_config: PretrainedConfig,
|
||||
) -> PretrainedConfig:
|
||||
hf_config = SpeculativeConfig.hf_config_override(hf_config)
|
||||
return target_hf_overrides(hf_config)
|
||||
|
||||
@staticmethod
|
||||
def compose_draft_hf_overrides(
|
||||
target_hf_overrides: HfOverrides | None,
|
||||
) -> Callable[[PretrainedConfig], PretrainedConfig]:
|
||||
"""Build the ``hf_overrides`` for the draft ``ModelConfig``.
|
||||
|
||||
Callable overrides on the target are config-to-config transforms
|
||||
(e.g. test harnesses shrinking ``num_hidden_layers``) and must also
|
||||
reach the draft config — otherwise a draft belonging to a large
|
||||
target is instantiated at full size even when the target is shrunk.
|
||||
Dict overrides are target-specific key patches and are not applied
|
||||
to the draft.
|
||||
|
||||
The composed override must stay picklable: the draft ``ModelConfig``
|
||||
is sent to spawned engine-core processes, so a local closure would
|
||||
fail with ``Can't get local object`` during pickling. Bind the
|
||||
target via ``functools.partial`` over a module-referenceable static
|
||||
method instead.
|
||||
"""
|
||||
if not callable(target_hf_overrides):
|
||||
return SpeculativeConfig.hf_config_override
|
||||
|
||||
return functools.partial(
|
||||
SpeculativeConfig._apply_composed_hf_override, target_hf_overrides
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
# Note: "method" is a new parameter that helps to extend the
|
||||
# configuration of non-model-based proposers, and the "model" parameter
|
||||
@@ -752,7 +788,12 @@ class SpeculativeConfig:
|
||||
if self.method == "medusa":
|
||||
draft_hf_overrides = {"model_type": "medusa"}
|
||||
else:
|
||||
draft_hf_overrides = SpeculativeConfig.hf_config_override
|
||||
# Compose any callable hf_overrides set on the target so the
|
||||
# draft config receives the same transform (e.g. the test
|
||||
# shrink). Dict overrides stay target-only.
|
||||
draft_hf_overrides = SpeculativeConfig.compose_draft_hf_overrides(
|
||||
self.target_model_config.hf_overrides
|
||||
)
|
||||
self.draft_model_config = ModelConfig(
|
||||
model=self.model,
|
||||
runner="draft",
|
||||
|
||||
Reference in New Issue
Block a user