From 727971f1c1056f7b8a3ea89713526756e6f7c8cf Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 30 Jun 2026 15:02:22 -0300 Subject: [PATCH] Add Medusa speculative decoding e2e test (#41396) Signed-off-by: Anshika Ojha Signed-off-by: Rishi Puri Signed-off-by: Rishi Puri Signed-off-by: Stefano Castagnetta Co-authored-by: Anshika Ojha <215760622+ojhaanshika@users.noreply.github.com> Co-authored-by: Claude Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Stefano Castagnetta --- tests/v1/e2e/spec_decode/test_spec_decode.py | 41 ++++++++++++++++++++ vllm/config/speculative.py | 24 +++++++++++- vllm/model_executor/models/medusa.py | 21 ++++++++++ vllm/transformers_utils/config.py | 2 +- 4 files changed, 85 insertions(+), 3 deletions(-) diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index 532a3e8d6a..0b94836cb0 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -718,6 +718,47 @@ def test_eagle_correctness_heavy( ) +@large_gpu_mark(min_gb=24) +def test_medusa_acceptance_rate( + sampling_config: SamplingParams, +): + """Verify a trained Medusa checkpoint achieves nonzero acceptance rate. + + Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the + speculation path actually accepts tokens — unlike test_medusa_correctness, + which uses a random head and only validates output correctness. + """ + target_model = "lmsys/vicuna-7b-v1.3" + medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3" + prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0] + + spec_llm = LLM( + model=target_model, + speculative_config={ + "method": "medusa", + "model": medusa_model, + "num_speculative_tokens": 3, + }, + max_model_len=1024, + enforce_eager=True, + disable_log_stats=False, + ) + spec_llm.generate(prompts, sampling_config) + metrics = spec_llm.get_metrics() + acceptance_rate = compute_acceptance_rate(metrics) + del spec_llm + torch.accelerator.empty_cache() + cleanup_dist_env_and_memory() + + min_acceptance_rate = 0.198 + print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})") + + # Regression guard at 90% of the measured baseline. + assert acceptance_rate >= min_acceptance_rate, ( + f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}" + ) + + @pytest.mark.parametrize( ["model_setup", "mm_enabled", "expected_accuracy_threshold"], [ diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 52464753ef..0a3347fd23 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -9,7 +9,7 @@ from typing_extensions import Self from vllm.config import LoadConfig from vllm.config.kernel import MoEBackend -from vllm.config.model import ModelConfig +from vllm.config.model import HfOverrides, ModelConfig from vllm.config.parallel import ParallelConfig from vllm.config.utils import config from vllm.logger import init_logger @@ -712,6 +712,15 @@ class SpeculativeConfig: self.prompt_lookup_min = 0 if self.model is not None: + # Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*) + # lack a model_type key in config.json, so AutoConfig cannot + # detect them. When the method is explicitly "medusa", inject + # model_type so MedusaConfig.from_pretrained is used instead. + draft_hf_overrides: HfOverrides + if self.method == "medusa": + draft_hf_overrides = {"model_type": "medusa"} + else: + draft_hf_overrides = SpeculativeConfig.hf_config_override self.draft_model_config = ModelConfig( model=self.model, runner="draft", @@ -730,10 +739,21 @@ class SpeculativeConfig: quantization=self.quantization, enforce_eager=self.target_model_config.enforce_eager, max_logprobs=self.target_model_config.max_logprobs, - hf_overrides=SpeculativeConfig.hf_config_override, + hf_overrides=draft_hf_overrides, config_format=self.target_model_config.config_format, ) + # Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*) + # omit vocab_size in config.json, so MedusaConfig falls back to + # its default (32001). Align with the target model's vocab size + # to avoid shape mismatches when loading LM-head weights. + if self.method == "medusa": + target_vocab = self.target_model_config.hf_config.vocab_size + draft_hf = self.draft_model_config.hf_config + if draft_hf.vocab_size != target_vocab: + draft_hf.vocab_size = target_vocab + draft_hf.truncated_vocab_size = target_vocab + # Automatically detect the method if self.method in ("eagle", "eagle3", "dflash"): pass diff --git a/vllm/model_executor/models/medusa.py b/vllm/model_executor/models/medusa.py index fd7fc2c73f..a857533ab4 100644 --- a/vllm/model_executor/models/medusa.py +++ b/vllm/model_executor/models/medusa.py @@ -136,6 +136,26 @@ class Medusa(nn.Module): return logits_lst + @staticmethod + def _remap_old_checkpoint_key(name: str) -> str: + """Map old FasterDecoding checkpoint keys to vLLM parameter names. + + Old format uses bare numeric prefixes: + '{head}.{layer}.linear.weight' -> 'blocks.{head}.layers.{layer}.weight' + '{head}.{layer}.linear.bias' -> 'blocks.{head}.layers.{layer}.bias' + '{head}.{N}.weight' -> 'lm_heads.{head}.weight' + """ + parts = name.split(".") + if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit(): + head, layer = parts[0], parts[1] + rest = parts[2:] + if "linear" in rest: + param = ".".join(rest[rest.index("linear") + 1 :]) + return f"blocks.{head}.layers.{layer}.{param}" + else: + return f"lm_heads.{head}.{'.'.join(rest)}" + return name + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -144,6 +164,7 @@ class Medusa(nn.Module): for name, loaded_weight in weights: name = name.replace("medusa_heads.", "") + name = self._remap_old_checkpoint_key(name) if name == "token_map": if self.truncated_vocab_size < self.orig_vocab_size: diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 67e3db93b8..ff1251665a 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -126,7 +126,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( **{"unlimited-ocr": "UnlimitedOCRConfig"}, ) -_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"} +_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"} _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}