Add Medusa speculative decoding e2e test (#41396)

Signed-off-by: Anshika Ojha <anshikao@nvidia.com>
Signed-off-by: Rishi Puri <riship@nvidia.com>
Signed-off-by: Rishi Puri <puririshi98@berkeley.edu>
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Anshika Ojha <215760622+ojhaanshika@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
Rishi Puri
2026-06-30 18:02:22 +00:00
committed by GitHub
co-authored by Anshika Ojha Claude gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Stefano Castagnetta
parent 25671cb520
commit 727971f1c1
4 changed files with 85 additions and 3 deletions
@@ -718,6 +718,47 @@ def test_eagle_correctness_heavy(
)
@large_gpu_mark(min_gb=24)
def test_medusa_acceptance_rate(
sampling_config: SamplingParams,
):
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
speculation path actually accepts tokens — unlike test_medusa_correctness,
which uses a random head and only validates output correctness.
"""
target_model = "lmsys/vicuna-7b-v1.3"
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
spec_llm = LLM(
model=target_model,
speculative_config={
"method": "medusa",
"model": medusa_model,
"num_speculative_tokens": 3,
},
max_model_len=1024,
enforce_eager=True,
disable_log_stats=False,
)
spec_llm.generate(prompts, sampling_config)
metrics = spec_llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
min_acceptance_rate = 0.198
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
# Regression guard at 90% of the measured baseline.
assert acceptance_rate >= min_acceptance_rate, (
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
)
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[