Add Medusa speculative decoding e2e test (#41396)

Signed-off-by: Anshika Ojha <anshikao@nvidia.com>
Signed-off-by: Rishi Puri <riship@nvidia.com>
Signed-off-by: Rishi Puri <puririshi98@berkeley.edu>
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Anshika Ojha <215760622+ojhaanshika@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
Rishi Puri
2026-06-30 18:02:22 +00:00
committed by GitHub
co-authored by Anshika Ojha Claude gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Stefano Castagnetta
parent 25671cb520
commit 727971f1c1
4 changed files with 85 additions and 3 deletions
@@ -718,6 +718,47 @@ def test_eagle_correctness_heavy(
)
@large_gpu_mark(min_gb=24)
def test_medusa_acceptance_rate(
sampling_config: SamplingParams,
):
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
speculation path actually accepts tokens — unlike test_medusa_correctness,
which uses a random head and only validates output correctness.
"""
target_model = "lmsys/vicuna-7b-v1.3"
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
spec_llm = LLM(
model=target_model,
speculative_config={
"method": "medusa",
"model": medusa_model,
"num_speculative_tokens": 3,
},
max_model_len=1024,
enforce_eager=True,
disable_log_stats=False,
)
spec_llm.generate(prompts, sampling_config)
metrics = spec_llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
min_acceptance_rate = 0.198
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
# Regression guard at 90% of the measured baseline.
assert acceptance_rate >= min_acceptance_rate, (
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
)
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
+22 -2
View File
@@ -9,7 +9,7 @@ from typing_extensions import Self
from vllm.config import LoadConfig
from vllm.config.kernel import MoEBackend
from vllm.config.model import ModelConfig
from vllm.config.model import HfOverrides, ModelConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.utils import config
from vllm.logger import init_logger
@@ -712,6 +712,15 @@ class SpeculativeConfig:
self.prompt_lookup_min = 0
if self.model is not None:
# Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*)
# lack a model_type key in config.json, so AutoConfig cannot
# detect them. When the method is explicitly "medusa", inject
# model_type so MedusaConfig.from_pretrained is used instead.
draft_hf_overrides: HfOverrides
if self.method == "medusa":
draft_hf_overrides = {"model_type": "medusa"}
else:
draft_hf_overrides = SpeculativeConfig.hf_config_override
self.draft_model_config = ModelConfig(
model=self.model,
runner="draft",
@@ -730,10 +739,21 @@ class SpeculativeConfig:
quantization=self.quantization,
enforce_eager=self.target_model_config.enforce_eager,
max_logprobs=self.target_model_config.max_logprobs,
hf_overrides=SpeculativeConfig.hf_config_override,
hf_overrides=draft_hf_overrides,
config_format=self.target_model_config.config_format,
)
# Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*)
# omit vocab_size in config.json, so MedusaConfig falls back to
# its default (32001). Align with the target model's vocab size
# to avoid shape mismatches when loading LM-head weights.
if self.method == "medusa":
target_vocab = self.target_model_config.hf_config.vocab_size
draft_hf = self.draft_model_config.hf_config
if draft_hf.vocab_size != target_vocab:
draft_hf.vocab_size = target_vocab
draft_hf.truncated_vocab_size = target_vocab
# Automatically detect the method
if self.method in ("eagle", "eagle3", "dflash"):
pass
+21
View File
@@ -136,6 +136,26 @@ class Medusa(nn.Module):
return logits_lst
@staticmethod
def _remap_old_checkpoint_key(name: str) -> str:
"""Map old FasterDecoding checkpoint keys to vLLM parameter names.
Old format uses bare numeric prefixes:
'{head}.{layer}.linear.weight' -> 'blocks.{head}.layers.{layer}.weight'
'{head}.{layer}.linear.bias' -> 'blocks.{head}.layers.{layer}.bias'
'{head}.{N}.weight' -> 'lm_heads.{head}.weight'
"""
parts = name.split(".")
if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
head, layer = parts[0], parts[1]
rest = parts[2:]
if "linear" in rest:
param = ".".join(rest[rest.index("linear") + 1 :])
return f"blocks.{head}.layers.{layer}.{param}"
else:
return f"lm_heads.{head}.{'.'.join(rest)}"
return name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -144,6 +164,7 @@ class Medusa(nn.Module):
for name, loaded_weight in weights:
name = name.replace("medusa_heads.", "")
name = self._remap_old_checkpoint_key(name)
if name == "token_map":
if self.truncated_vocab_size < self.orig_vocab_size:
+1 -1
View File
@@ -126,7 +126,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
**{"unlimited-ocr": "UnlimitedOCRConfig"},
)
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"}
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"}
_PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}