[CI] Solidify speculative decoding E2E coverage (#50713)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-10 19:56:39 -06:00
committed by GitHub
co-authored by OpenAI Codex
parent 48bada6ea4
commit 1a1727330a
18 changed files with 737 additions and 459 deletions
+48 -46
View File
@@ -4,15 +4,15 @@
from typing import Any
import pytest
import torch
from tests.utils import single_gpu_only
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm import SamplingParams
from vllm.config import CompilationConfig
from vllm.platforms import current_platform
from ..utils import (
_skip_if_insufficient_gpus_for_tp,
assert_request_outputs_match,
evaluate_llm_for_gsm8k,
get_test_prompts,
)
@@ -21,9 +21,10 @@ from ..utils import (
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1), False, 0.5), # ref: 65%-70%
# Reference accuracy: 65%-70%.
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
pytest.param(
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1),
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
False,
0.0,
marks=pytest.mark.skipif(
@@ -32,7 +33,7 @@ from ..utils import (
),
), # dummy model
(
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1),
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
False,
0.20,
), # hybrid + MTP, ref: ~34%-35%
@@ -48,9 +49,10 @@ from ..utils import (
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int] | tuple[str, str, int, str],
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
"""
Compare the outputs of a original LLM and a speculative LLM
@@ -59,18 +61,14 @@ def test_mtp_correctness(
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
reference threshold for each model.
"""
method, model_name, tp_size, draft_model = model_setup
_skip_if_insufficient_gpus_for_tp(tp_size)
# Generate test prompts inside the function instead of using fixture
test_prompts = get_test_prompts(mm_enabled)
with monkeypatch.context() as m:
m.setenv("VLLM_MLA_DISABLE", "1")
if len(model_setup) == 4:
method, model_name, tp_size, draft_model = model_setup
else:
method, model_name, tp_size = model_setup
draft_model = None
_skip_if_insufficient_gpus_for_tp(tp_size)
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
# Skip multimodal profiling for models that don't need it in this test.
@@ -90,21 +88,22 @@ def test_mtp_correctness(
f"got {transformers.__version__}"
)
ref_llm = LLM(
model=model_name,
with vllm_runner(
model_name,
block_size=None,
max_model_len=2048,
tensor_parallel_size=tp_size,
trust_remote_code=True,
attention_backend=attn_backend,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
)
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
evaluate_llm_for_gsm8k(
ref_llm, expected_accuracy_threshold=expected_accuracy_threshold
)
del ref_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
) as ref_runner:
ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
evaluate_llm_for_gsm8k(
ref_runner.llm,
expected_accuracy_threshold=expected_accuracy_threshold,
)
speculative_config: dict[str, Any] = {
"method": method,
@@ -115,34 +114,37 @@ def test_mtp_correctness(
speculative_config["model"] = draft_model
speculative_config["num_speculative_tokens"] = 2
spec_llm = LLM(
model=model_name,
with vllm_runner(
model_name,
block_size=None,
trust_remote_code=True,
tensor_parallel_size=tp_size,
speculative_config=speculative_config,
max_model_len=2048,
attention_backend=attn_backend,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
)
# MTP supports async scheduling; assert it is active by default.
assert spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
evaluate_llm_for_gsm8k(
spec_llm, expected_accuracy_threshold=expected_accuracy_threshold
)
spec_outputs = spec_llm.chat(test_prompts, sampling_config)
matches = 0
misses = 0
for ref_output, spec_output in zip(ref_outputs, spec_outputs):
if ref_output.outputs[0].text == spec_output.outputs[0].text:
matches += 1
else:
misses += 1
print(f"ref_output: {ref_output.outputs[0].text}")
print(f"spec_output: {spec_output.outputs[0].text}")
) as spec_runner:
# MTP supports async scheduling by default.
has_async = (
spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
)
assert has_async, (
f"Expected async scheduling for {method}: target={model_name}, "
f"draft={draft_model}; got {has_async}"
)
evaluate_llm_for_gsm8k(
spec_runner.llm,
expected_accuracy_threshold=expected_accuracy_threshold,
)
spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
# Heuristic: expect at least 80% of the prompts to match exactly
# Upon failure, inspect the outputs to check for inaccuracy.
assert matches > int(0.8 * len(ref_outputs))
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
assert_request_outputs_match(
ref_outputs,
spec_outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context=f"{method} target={model_name}, draft={draft_model}",
)