Add Medusa speculative decoding e2e test (#41396)
Signed-off-by: Anshika Ojha <anshikao@nvidia.com> Signed-off-by: Rishi Puri <riship@nvidia.com> Signed-off-by: Rishi Puri <puririshi98@berkeley.edu> Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Anshika Ojha <215760622+ojhaanshika@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
co-authored by
Anshika Ojha
Claude
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Stefano Castagnetta
parent
25671cb520
commit
727971f1c1
@@ -718,6 +718,47 @@ def test_eagle_correctness_heavy(
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=24)
|
||||
def test_medusa_acceptance_rate(
|
||||
sampling_config: SamplingParams,
|
||||
):
|
||||
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
|
||||
|
||||
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
|
||||
speculation path actually accepts tokens — unlike test_medusa_correctness,
|
||||
which uses a random head and only validates output correctness.
|
||||
"""
|
||||
target_model = "lmsys/vicuna-7b-v1.3"
|
||||
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
|
||||
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
|
||||
|
||||
spec_llm = LLM(
|
||||
model=target_model,
|
||||
speculative_config={
|
||||
"method": "medusa",
|
||||
"model": medusa_model,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
spec_llm.generate(prompts, sampling_config)
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
min_acceptance_rate = 0.198
|
||||
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
|
||||
|
||||
# Regression guard at 90% of the measured baseline.
|
||||
assert acceptance_rate >= min_acceptance_rate, (
|
||||
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing_extensions import Self
|
||||
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.model import HfOverrides, ModelConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
@@ -712,6 +712,15 @@ class SpeculativeConfig:
|
||||
self.prompt_lookup_min = 0
|
||||
|
||||
if self.model is not None:
|
||||
# Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*)
|
||||
# lack a model_type key in config.json, so AutoConfig cannot
|
||||
# detect them. When the method is explicitly "medusa", inject
|
||||
# model_type so MedusaConfig.from_pretrained is used instead.
|
||||
draft_hf_overrides: HfOverrides
|
||||
if self.method == "medusa":
|
||||
draft_hf_overrides = {"model_type": "medusa"}
|
||||
else:
|
||||
draft_hf_overrides = SpeculativeConfig.hf_config_override
|
||||
self.draft_model_config = ModelConfig(
|
||||
model=self.model,
|
||||
runner="draft",
|
||||
@@ -730,10 +739,21 @@ class SpeculativeConfig:
|
||||
quantization=self.quantization,
|
||||
enforce_eager=self.target_model_config.enforce_eager,
|
||||
max_logprobs=self.target_model_config.max_logprobs,
|
||||
hf_overrides=SpeculativeConfig.hf_config_override,
|
||||
hf_overrides=draft_hf_overrides,
|
||||
config_format=self.target_model_config.config_format,
|
||||
)
|
||||
|
||||
# Old-format Medusa checkpoints (e.g. FasterDecoding/medusa-*)
|
||||
# omit vocab_size in config.json, so MedusaConfig falls back to
|
||||
# its default (32001). Align with the target model's vocab size
|
||||
# to avoid shape mismatches when loading LM-head weights.
|
||||
if self.method == "medusa":
|
||||
target_vocab = self.target_model_config.hf_config.vocab_size
|
||||
draft_hf = self.draft_model_config.hf_config
|
||||
if draft_hf.vocab_size != target_vocab:
|
||||
draft_hf.vocab_size = target_vocab
|
||||
draft_hf.truncated_vocab_size = target_vocab
|
||||
|
||||
# Automatically detect the method
|
||||
if self.method in ("eagle", "eagle3", "dflash"):
|
||||
pass
|
||||
|
||||
@@ -136,6 +136,26 @@ class Medusa(nn.Module):
|
||||
|
||||
return logits_lst
|
||||
|
||||
@staticmethod
|
||||
def _remap_old_checkpoint_key(name: str) -> str:
|
||||
"""Map old FasterDecoding checkpoint keys to vLLM parameter names.
|
||||
|
||||
Old format uses bare numeric prefixes:
|
||||
'{head}.{layer}.linear.weight' -> 'blocks.{head}.layers.{layer}.weight'
|
||||
'{head}.{layer}.linear.bias' -> 'blocks.{head}.layers.{layer}.bias'
|
||||
'{head}.{N}.weight' -> 'lm_heads.{head}.weight'
|
||||
"""
|
||||
parts = name.split(".")
|
||||
if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
|
||||
head, layer = parts[0], parts[1]
|
||||
rest = parts[2:]
|
||||
if "linear" in rest:
|
||||
param = ".".join(rest[rest.index("linear") + 1 :])
|
||||
return f"blocks.{head}.layers.{layer}.{param}"
|
||||
else:
|
||||
return f"lm_heads.{head}.{'.'.join(rest)}"
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -144,6 +164,7 @@ class Medusa(nn.Module):
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
name = name.replace("medusa_heads.", "")
|
||||
name = self._remap_old_checkpoint_key(name)
|
||||
|
||||
if name == "token_map":
|
||||
if self.truncated_vocab_size < self.orig_vocab_size:
|
||||
|
||||
@@ -126,7 +126,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
||||
)
|
||||
|
||||
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"}
|
||||
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"}
|
||||
|
||||
_PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user