[Spec Decode] Remove Medusa and MLP speculator support (#61701)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
273a04964e
commit
8530aed2b7
@@ -1,21 +1,4 @@
|
||||
{
|
||||
"abhigoyal/vllm-medusa-llama-68m-random": {
|
||||
"architectures": [
|
||||
"MedusaModel"
|
||||
],
|
||||
"model_type": "medusa",
|
||||
"text_model_type": "medusa",
|
||||
"hidden_size": 768,
|
||||
"total_num_hidden_layers": 1,
|
||||
"total_num_attention_heads": 0,
|
||||
"head_size": "Error: integer division or modulo by zero",
|
||||
"vocab_size": 32000,
|
||||
"total_num_kv_heads": 0,
|
||||
"num_experts": 0,
|
||||
"is_deepseek_mla": false,
|
||||
"is_multimodal_model": false,
|
||||
"dtype": "torch.float32"
|
||||
},
|
||||
"luccafong/deepseek_mtp_draft_random": {
|
||||
"architectures": [
|
||||
"DeepSeekMTPModel"
|
||||
|
||||
@@ -55,7 +55,6 @@ BASE_MODELS_TO_TEST = [
|
||||
|
||||
# (target_model, draft_model, trust_remote_code)
|
||||
SPECULATIVE_MODELS = [
|
||||
("JackFram/llama-68m", "abhigoyal/vllm-medusa-llama-68m-random", False),
|
||||
("luccafong/deepseek_mtp_main_random", "luccafong/deepseek_mtp_draft_random", True),
|
||||
("eagle618/deepseek-v3-random", "eagle618/eagle-deepseek-v3-random", True),
|
||||
("meta-llama/Meta-Llama-3-8B-Instruct", "yuhuili/EAGLE-LLaMA3-Instruct-8B", True),
|
||||
@@ -136,9 +135,7 @@ def test_qwen35_decision_config_selects_d3(tmp_path, has_decision_config, archit
|
||||
assert not hasattr(config, "decision_config")
|
||||
|
||||
|
||||
def _assert_model_arch_config(
|
||||
model_config, expected: dict, check_head_size: bool = True
|
||||
):
|
||||
def _assert_model_arch_config(model_config, expected: dict):
|
||||
"""Assert model_arch_config matches expected values."""
|
||||
model_arch_config = model_config.model_arch_config
|
||||
assert model_arch_config.architectures == expected["architectures"]
|
||||
@@ -164,14 +161,10 @@ def _assert_model_arch_config(
|
||||
config_format="hf",
|
||||
)
|
||||
assert str(torch_dtype) == expected["dtype"]
|
||||
|
||||
if check_head_size:
|
||||
assert model_arch_config.head_size == expected["head_size"]
|
||||
assert model_arch_config.head_size == expected["head_size"]
|
||||
|
||||
|
||||
def _assert_model_config_methods(
|
||||
model_config, expected: dict, check_head_size: bool = True
|
||||
):
|
||||
def _assert_model_config_methods(model_config, expected: dict):
|
||||
"""Assert model_config methods return expected values."""
|
||||
assert model_config.architectures == expected["architectures"]
|
||||
assert model_config.get_vocab_size() == expected["vocab_size"]
|
||||
@@ -182,9 +175,7 @@ def _assert_model_config_methods(
|
||||
model_config.get_total_num_hidden_layers()
|
||||
== expected["total_num_hidden_layers"]
|
||||
)
|
||||
|
||||
if check_head_size:
|
||||
assert model_config.get_head_size() == expected["head_size"]
|
||||
assert model_config.get_head_size() == expected["head_size"]
|
||||
|
||||
|
||||
def test_head_size_falls_back_when_head_dim_is_zero():
|
||||
@@ -519,14 +510,8 @@ def test_draft_model_arch_config(
|
||||
)
|
||||
model_config = speculative_config.draft_model_config
|
||||
|
||||
# For medusa models, head_size may cause division by zero before
|
||||
# model_arch_config was introduced, so we conditionally check it
|
||||
check_head_size = isinstance(expected["head_size"], int)
|
||||
|
||||
_assert_model_arch_config(model_config, expected, check_head_size=check_head_size)
|
||||
_assert_model_config_methods(
|
||||
model_config, expected, check_head_size=check_head_size
|
||||
)
|
||||
_assert_model_arch_config(model_config, expected)
|
||||
_assert_model_config_methods(model_config, expected)
|
||||
|
||||
|
||||
def test_deepseek_v4_convertor_splits_vision_architecture():
|
||||
|
||||
@@ -1456,16 +1456,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
|
||||
|
||||
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
# [Medusa]
|
||||
"MedusaModel": _HfExamplesInfo(
|
||||
"JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
|
||||
),
|
||||
# Temporarily disabled.
|
||||
# TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
# "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
|
||||
# "JackFram/llama-160m",
|
||||
# speculative_model="ibm-ai-platform/llama-160m-accelerator"
|
||||
# ),
|
||||
# [DFlash]
|
||||
"DFlashDraftModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-4B",
|
||||
|
||||
@@ -113,9 +113,6 @@ def test_registry_model_property(model_arch, is_mm, score_type):
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,is_pp",
|
||||
[
|
||||
# TODO(woosuk): Re-enable this once the MLP Speculator is supported
|
||||
# in V1.
|
||||
# ("MLPSpeculatorPreTrainedModel", False),
|
||||
("DeepseekV2ForCausalLM", True),
|
||||
("Qwen2VLForConditionalGeneration", True),
|
||||
],
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
|
||||
from ...utils import compute_acceptance_rate
|
||||
|
||||
|
||||
def test_medusa_acceptance_rate(
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
|
||||
|
||||
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
|
||||
speculation path actually accepts tokens — unlike test_medusa_correctness,
|
||||
which uses a random head and only validates output correctness.
|
||||
"""
|
||||
target_model = "lmsys/vicuna-7b-v1.3"
|
||||
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
|
||||
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
|
||||
|
||||
with vllm_runner(
|
||||
target_model,
|
||||
block_size=None,
|
||||
trust_remote_code=False,
|
||||
speculative_config={
|
||||
"method": "medusa",
|
||||
"model": medusa_model,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=None,
|
||||
disable_log_stats=False,
|
||||
compilation_config=CompilationConfig(),
|
||||
) as spec_runner:
|
||||
spec_runner.llm.generate(prompts, sampling_config)
|
||||
metrics = spec_runner.llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
|
||||
min_acceptance_rate = 0.198
|
||||
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
|
||||
|
||||
# Regression guard at 90% of the measured baseline.
|
||||
assert acceptance_rate >= min_acceptance_rate, (
|
||||
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
|
||||
)
|
||||
@@ -334,8 +334,6 @@ def test_reserved_block_count_matches_real_kv_cache_manager():
|
||||
("dflash", None, NUM_SPEC_STEPS + 1),
|
||||
("ngram", None, 0),
|
||||
("ngram_gpu", None, 0),
|
||||
("medusa", None, 0),
|
||||
("mlp_speculator", None, 0),
|
||||
("suffix", None, 0),
|
||||
("extract_hidden_states", None, 0),
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user