[CI] Organize speculative decoding E2E tests by coverage (#50330)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Michael Goin
2026-07-31 18:05:42 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b40d859c7b
commit fcdc7c2e9c
39 changed files with 1941 additions and 1863 deletions
+155
View File
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import Any
import pytest
import torch
from tests.utils import single_gpu_only
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
from ..utils import (
_skip_if_insufficient_gpus_for_tp,
evaluate_llm_for_gsm8k,
get_test_prompts,
)
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1), False, 0.5), # ref: 65%-70%
pytest.param(
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1),
False,
0.0,
marks=pytest.mark.skipif(
current_platform.is_device_capability_family(100),
reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
),
), # dummy model
(
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1),
False,
0.20,
), # hybrid + MTP, ref: ~34%-35%
(
("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
False,
0.50,
), # gemma4 MTP with assistant model, ref: ~62%
],
ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"],
)
@single_gpu_only
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int] | tuple[str, str, int, str],
mm_enabled: bool,
expected_accuracy_threshold: float,
):
"""
Compare the outputs of a original LLM and a speculative LLM
which should be the same when using MTP speculative decoding. Due to some variance
in the engine, it is possible for some outputs to differ, so we expect that at least
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
reference threshold for each model.
"""
# Generate test prompts inside the function instead of using fixture
test_prompts = get_test_prompts(mm_enabled)
with monkeypatch.context() as m:
m.setenv("VLLM_MLA_DISABLE", "1")
if len(model_setup) == 4:
method, model_name, tp_size, draft_model = model_setup
else:
method, model_name, tp_size = model_setup
draft_model = None
_skip_if_insufficient_gpus_for_tp(tp_size)
if "Qwen3.5" in model_name and os.environ.get("VLLM_USE_V2_MODEL_RUNNER"):
pytest.skip(
"Model Runner V2 does not yet support hybrid models "
"(Qwen3.5 mixes Mamba-style GDN with attention layers)."
)
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
# Skip multimodal profiling for models that don't need it in this test.
extra_kwargs: dict[str, Any] = {}
if "Qwen3.5" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
elif "gemma-4" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
if draft_model is not None and "gemma-4" in draft_model:
import transformers
from packaging.version import Version
if Version(transformers.__version__) < Version("5.8.0"):
pytest.skip(
"Gemma4 MTP assistant requires transformers>=5.8.0, "
f"got {transformers.__version__}"
)
ref_llm = LLM(
model=model_name,
max_model_len=2048,
tensor_parallel_size=tp_size,
trust_remote_code=True,
attention_backend=attn_backend,
**extra_kwargs,
)
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
evaluate_llm_for_gsm8k(
ref_llm, expected_accuracy_threshold=expected_accuracy_threshold
)
del ref_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
speculative_config: dict[str, Any] = {
"method": method,
"num_speculative_tokens": 1,
"max_model_len": 2048,
}
if draft_model is not None:
speculative_config["model"] = draft_model
speculative_config["num_speculative_tokens"] = 2
spec_llm = LLM(
model=model_name,
trust_remote_code=True,
tensor_parallel_size=tp_size,
speculative_config=speculative_config,
max_model_len=2048,
attention_backend=attn_backend,
**extra_kwargs,
)
# MTP supports async scheduling; assert it is active by default.
assert spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
evaluate_llm_for_gsm8k(
spec_llm, expected_accuracy_threshold=expected_accuracy_threshold
)
spec_outputs = spec_llm.chat(test_prompts, sampling_config)
matches = 0
misses = 0
for ref_output, spec_output in zip(ref_outputs, spec_outputs):
if ref_output.outputs[0].text == spec_output.outputs[0].text:
matches += 1
else:
misses += 1
print(f"ref_output: {ref_output.outputs[0].text}")
print(f"spec_output: {spec_output.outputs[0].text}")
# Heuristic: expect at least 80% of the prompts to match exactly
# Upon failure, inspect the outputs to check for inaccuracy.
assert matches > int(0.8 * len(ref_outputs))
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()