Add Medusa speculative decoding e2e test (#41396)
Signed-off-by: Anshika Ojha <anshikao@nvidia.com> Signed-off-by: Rishi Puri <riship@nvidia.com> Signed-off-by: Rishi Puri <puririshi98@berkeley.edu> Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Anshika Ojha <215760622+ojhaanshika@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
co-authored by
Anshika Ojha
Claude
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Stefano Castagnetta
parent
25671cb520
commit
727971f1c1
@@ -718,6 +718,47 @@ def test_eagle_correctness_heavy(
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=24)
|
||||
def test_medusa_acceptance_rate(
|
||||
sampling_config: SamplingParams,
|
||||
):
|
||||
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
|
||||
|
||||
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
|
||||
speculation path actually accepts tokens — unlike test_medusa_correctness,
|
||||
which uses a random head and only validates output correctness.
|
||||
"""
|
||||
target_model = "lmsys/vicuna-7b-v1.3"
|
||||
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
|
||||
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
|
||||
|
||||
spec_llm = LLM(
|
||||
model=target_model,
|
||||
speculative_config={
|
||||
"method": "medusa",
|
||||
"model": medusa_model,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
spec_llm.generate(prompts, sampling_config)
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
min_acceptance_rate = 0.198
|
||||
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
|
||||
|
||||
# Regression guard at 90% of the measured baseline.
|
||||
assert acceptance_rate >= min_acceptance_rate, (
|
||||
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user