[CI] Solidify speculative decoding E2E coverage (#50713)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-10 19:56:39 -06:00
committed by GitHub
co-authored by OpenAI Codex
parent 48bada6ea4
commit 1a1727330a
18 changed files with 737 additions and 459 deletions
@@ -102,14 +102,15 @@ def test_no_sync_with_spec_decode(
spec_model: str,
method: str,
num_spec_tokens: int,
vllm_runner,
):
"""
Test that no implicit GPU-CPU sync occurs during speculative decoding
generation.
"""
# Import vLLM AFTER sync_tracker fixture has applied the patch
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm import SamplingParams
from vllm.config import CompilationConfig
# Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
# and peaks well above the 18GB MIG slice used by one of the CI lanes.
@@ -119,8 +120,10 @@ def test_no_sync_with_spec_decode(
if "Qwen3.5" in model:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
llm = LLM(
model=model,
with vllm_runner(
model,
block_size=None,
trust_remote_code=False,
max_model_len=256,
speculative_config={
"method": method,
@@ -129,24 +132,31 @@ def test_no_sync_with_spec_decode(
},
enforce_eager=True,
async_scheduling=True,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
)
) as runner:
llm = runner.llm
# Assert async scheduling is actually active before running inference.
assert llm.llm_engine.vllm_config.scheduler_config.async_scheduling, (
f"Expected async_scheduling=True for spec decode, got False. method={method}"
)
# Assert async scheduling is actually active before running inference.
assert llm.llm_engine.vllm_config.scheduler_config.async_scheduling, (
f"Expected async_scheduling=True for spec decode, got False. "
f"method={method}, target={model}, draft={spec_model}"
)
outputs = llm.generate(
["Hello, my name is"],
SamplingParams(temperature=0, max_tokens=10),
)
outputs = llm.generate(
["Hello, my name is"],
SamplingParams(temperature=0, max_tokens=10),
)
assert len(outputs) == 1
assert len(outputs[0].outputs[0].text) > 0
del llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
assert len(outputs) == 1, (
f"{method} target={model}: expected one request output, got {len(outputs)}"
)
assert outputs[0].outputs, (
f"{method} target={model}: request output has no completion candidates"
)
assert outputs[0].outputs[0].text, (
f"{method} target={model}: generated completion text is empty"
)
sync_tracker.assert_no_sync()
@@ -4,17 +4,16 @@
from dataclasses import dataclass
import pytest
import torch
from tests.utils import multi_gpu_only, single_gpu_only
from vllm import LLM, SamplingParams
from vllm.config import VllmConfig, replace
from vllm.distributed import cleanup_dist_env_and_memory
from vllm import SamplingParams
from vllm.config import CompilationConfig, VllmConfig, replace
from vllm.config.kernel import MoEBackend
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from ..utils import (
Messages,
_skip_if_insufficient_gpus_for_tp,
compute_acceptance_len,
compute_acceptance_rate,
evaluate_llm_for_gsm8k,
@@ -42,7 +41,8 @@ class ArgsTest:
max_model_len: int = 2048
gpu_memory_utilization: float = 0.5
dataset: str = "test_prompts"
num_prompts: int = 100
# Doubling 100 to 200 reduces sampling standard error by about 29%.
num_prompts: int = 200
def get_messages(dataset: str, n: int) -> list[Messages]:
@@ -89,13 +89,13 @@ cases = [
@pytest.mark.parametrize("args", cases)
@pytest.mark.parametrize("enforce_eager", [True, False])
@single_gpu_only
def test_draft_model_correctness(args: ArgsTest, enforce_eager: bool):
def test_draft_model_correctness(args: ArgsTest, enforce_eager: bool, vllm_runner):
args.enforce_eager = enforce_eager
assert_draft_model_correctness(args)
assert_draft_model_correctness(args, vllm_runner)
@single_gpu_only
def test_draft_model_realistic_example():
def test_draft_model_realistic_example(vllm_runner):
args = ArgsTest(
target_model="Qwen/Qwen3-1.7B",
draft_model="Qwen/Qwen3-0.6B",
@@ -106,11 +106,11 @@ def test_draft_model_realistic_example():
expected_acceptance_len=2.6, # ref: 2.86
expected_acceptance_rate=0.5, # ref: 0.62
)
assert_draft_model_correctness(args)
assert_draft_model_correctness(args, vllm_runner)
@single_gpu_only
def test_draft_model_parallel_drafting():
def test_draft_model_parallel_drafting(vllm_runner):
args = ArgsTest(
target_model="Qwen/Qwen3-1.7B",
draft_model="amd/PARD-Qwen3-0.6B",
@@ -122,7 +122,7 @@ def test_draft_model_parallel_drafting():
expected_acceptance_len=2.3, # ref: 2.52
expected_acceptance_rate=0.4, # ref: 0.51
)
assert_draft_model_correctness(args)
assert_draft_model_correctness(args, vllm_runner)
@pytest.mark.parametrize(
@@ -136,7 +136,9 @@ def test_draft_model_parallel_drafting():
)
@pytest.mark.parametrize("enforce_eager", [True, False])
@single_gpu_only
def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
def test_draft_model_quantization(
models: tuple[str, str], enforce_eager: bool, vllm_runner
):
tgt_model, draft_model = models
sd_case = ArgsTest(
target_model=tgt_model,
@@ -144,13 +146,12 @@ def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
**some_high_acceptance_metrics(),
enforce_eager=enforce_eager,
)
assert_draft_model_correctness(sd_case)
assert_draft_model_correctness(sd_case, vllm_runner)
@multi_gpu_only(num_gpus=2)
def test_draft_model_tensor_parallelism():
def test_draft_model_tensor_parallelism(vllm_runner):
"""Ensure spec decode works when running with TP > 1."""
_skip_if_insufficient_gpus_for_tp(2)
sd_case = ArgsTest(
target_model="Qwen/Qwen3-1.7B",
target_tensor_parallel_size=2,
@@ -160,15 +161,13 @@ def test_draft_model_tensor_parallelism():
enforce_eager=False,
expected_gsm8k_accuracy=0.5,
)
assert_draft_model_correctness(sd_case)
assert_draft_model_correctness(sd_case, vllm_runner)
@multi_gpu_only(num_gpus=2)
def test_draft_model_engine_args_tensor_parallelism():
"""Ensure the vllm_config for the draft model is created correctly,
and independently of the target model (quantization, TP, etc.)"""
_skip_if_insufficient_gpus_for_tp(2)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
tensor_parallel_size=2,
@@ -212,13 +211,24 @@ def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
return vllm_config
def test_draft_model_moe_backend_override():
def _platform_moe_backend(monkeypatch: pytest.MonkeyPatch) -> MoEBackend:
if current_platform.is_rocm():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
return "aiter"
if current_platform.is_cuda():
return "flashinfer_trtllm"
return "auto"
def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
"""When moe_backend is set in speculative_config, the draft VllmConfig
should use it while the target keeps its own setting."""
target_moe_backend = _platform_moe_backend(monkeypatch)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=1,
moe_backend="flashinfer_trtllm",
moe_backend=target_moe_backend,
speculative_config={
"model": "Qwen/Qwen3-0.6B",
"method": "draft_model",
@@ -227,22 +237,23 @@ def test_draft_model_moe_backend_override():
},
)
tgt_config: VllmConfig = engine_args.create_engine_config()
assert tgt_config.kernel_config.moe_backend == "flashinfer_trtllm"
assert tgt_config.kernel_config.moe_backend == target_moe_backend
assert tgt_config.speculative_config.moe_backend == "triton"
draft_config = _apply_draft_moe_backend(tgt_config)
assert draft_config.kernel_config.moe_backend == "triton"
# Target config must be unaffected.
assert tgt_config.kernel_config.moe_backend == "flashinfer_trtllm"
assert tgt_config.kernel_config.moe_backend == target_moe_backend
def test_draft_model_moe_backend_inherits_target():
def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch):
"""When moe_backend is not set in speculative_config, the draft should
inherit the target's moe_backend."""
target_moe_backend = _platform_moe_backend(monkeypatch)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=1,
moe_backend="flashinfer_cutlass",
moe_backend=target_moe_backend,
speculative_config={
"model": "Qwen/Qwen3-0.6B",
"method": "draft_model",
@@ -250,11 +261,11 @@ def test_draft_model_moe_backend_inherits_target():
},
)
tgt_config: VllmConfig = engine_args.create_engine_config()
assert tgt_config.kernel_config.moe_backend == "flashinfer_cutlass"
assert tgt_config.kernel_config.moe_backend == target_moe_backend
assert tgt_config.speculative_config.moe_backend is None
draft_config = _apply_draft_moe_backend(tgt_config)
assert draft_config.kernel_config.moe_backend == "flashinfer_cutlass"
assert draft_config.kernel_config.moe_backend == target_moe_backend
assert draft_config is tgt_config
@@ -293,19 +304,23 @@ def test_draft_model_engine_args_rejects_invalid_tp_argname():
"tensor_parallel_size": 1, # <<< invalid arg name
},
)
with pytest.raises(ValueError):
with pytest.raises(ValueError, match="draft_tensor_parallel_size"):
engine_args.create_engine_config()
def assert_draft_model_correctness(args: ArgsTest):
def assert_draft_model_correctness(args: ArgsTest, vllm_runner):
"""Compare the outputs using and not using speculative decoding.
In the greedy decoding case, the outputs must match EXACTLY."""
test_prompts: list[Messages] = get_messages(
dataset=args.dataset, n=args.num_prompts
)
spec_llm = LLM(
model=args.target_model,
with vllm_runner(
args.target_model,
block_size=None,
trust_remote_code=False,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
speculative_config={
"model": args.draft_model,
"method": "draft_model",
@@ -321,30 +336,41 @@ def assert_draft_model_correctness(args: ArgsTest):
tensor_parallel_size=args.target_tensor_parallel_size,
enforce_eager=args.enforce_eager,
disable_log_stats=False, # enables get_metrics()
)
) as spec_runner:
spec_llm = spec_runner.llm
# we don't check the outputs, only check the metrics
spec_llm.chat(test_prompts, args.sampling_config)
metrics = spec_llm.get_metrics()
acceptance_rate: float = compute_acceptance_rate(metrics)
acceptance_len: float = compute_acceptance_len(metrics)
# we don't check the outputs, only check the metrics
spec_llm.chat(test_prompts, args.sampling_config)
metrics = spec_llm.get_metrics()
acceptance_rate: float = compute_acceptance_rate(metrics)
acceptance_len: float = compute_acceptance_len(metrics)
# Need to evaluate after getting metrics to avoid polluting the AR
evaluate_llm_for_gsm8k(
spec_llm, expected_accuracy_threshold=args.expected_gsm8k_accuracy
)
# Need to evaluate after getting metrics to avoid polluting the AR
evaluate_llm_for_gsm8k(
spec_llm, expected_accuracy_threshold=args.expected_gsm8k_accuracy
)
print(
f"spec-decode: target={args.target_model}, draft={args.draft_model}, "
f"temperature={args.sampling_config.temperature:.2f}, "
f"acceptance_rate={acceptance_rate:.2f}, "
f"acceptance_len={acceptance_len:.2f}, "
)
print(
f"spec-decode: target={args.target_model}, draft={args.draft_model}, "
f"temperature={args.sampling_config.temperature:.2f}, "
f"acceptance_rate={acceptance_rate:.2f}, "
f"acceptance_len={acceptance_len:.2f}, "
)
context = (
f"draft_model target={args.target_model}, draft={args.draft_model}, "
f"eager={args.enforce_eager}, "
f"acceptance_rate={acceptance_rate:.3f}, "
f"acceptance_len={acceptance_len:.3f}"
)
assert acceptance_rate >= args.expected_acceptance_rate, (
f"{context}; expected acceptance_rate >= "
f"{args.expected_acceptance_rate:.3f}"
)
assert acceptance_len >= args.expected_acceptance_len, (
f"{context}; expected acceptance_len >= {args.expected_acceptance_len:.3f}"
)
# draft_model supports async scheduling; assert it is active by default.
has_async = spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
assert acceptance_rate >= args.expected_acceptance_rate
assert acceptance_len >= args.expected_acceptance_len
has_async = spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
del spec_llm # CLEANUP
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
assert has_async, "Expected async_scheduling=True for draft_model spec decode"
@@ -8,12 +8,14 @@ This script contains:
import pytest
import torch
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm import SamplingParams
from vllm.config import CompilationConfig
from vllm.lora.request import LoRARequest
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from ..utils import assert_request_outputs_match
LORA_TEST_PROMPT_MAP: dict[str, str] = {}
LORA_TEST_PROMPT_MAP["premjatin/qwen-linear-algebra-coder"] = """
@@ -36,7 +38,6 @@ Numbers should be represented as integers only.
SEED = 42
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
@pytest.mark.parametrize(
"model_setup",
[
@@ -49,9 +50,13 @@ SEED = 42
)
],
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
)
def test_batch_inference_correctness(
monkeypatch: pytest.MonkeyPatch,
model_setup: tuple[str, str, str, str, int],
vllm_runner,
):
"""
Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
@@ -60,10 +65,11 @@ def test_batch_inference_correctness(
"""
with monkeypatch.context() as m:
# Disable randomness
m.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
if current_platform.is_cuda():
m.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
set_random_seed(SEED)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
m.setattr(torch.backends.cudnn, "benchmark", False)
m.setattr(torch.backends.cudnn, "deterministic", True)
method, model_name, spec_model_name, lora_path, tp_size = model_setup
@@ -74,8 +80,9 @@ def test_batch_inference_correctness(
)
# without speculative decoding
ref_llm = LLM(
model=model_name,
with vllm_runner(
model_name,
block_size=None,
trust_remote_code=True,
tensor_parallel_size=tp_size,
max_model_len=2048,
@@ -84,18 +91,16 @@ def test_batch_inference_correctness(
max_loras=1,
max_cpu_loras=1,
max_lora_rank=16,
)
try:
ref_outputs = ref_llm.generate(
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
) as ref_runner:
ref_outputs = ref_runner.llm.generate(
prompts, sampling_params, lora_request=lora_request
)
finally:
del ref_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
lora_spec_llm = LLM(
model=model_name,
with vllm_runner(
model_name,
block_size=None,
trust_remote_code=True,
tensor_parallel_size=tp_size,
speculative_config={
@@ -110,30 +115,22 @@ def test_batch_inference_correctness(
max_loras=1,
max_cpu_loras=1,
max_lora_rank=16,
)
try:
lora_spec_outputs = lora_spec_llm.generate(
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
) as spec_runner:
lora_spec_outputs = spec_runner.llm.generate(
prompts, sampling_params, lora_request=lora_request
)
matches = 0
for ref_output, spec_output in zip(ref_outputs, lora_spec_outputs):
if ref_output.outputs[0].text == spec_output.outputs[0].text:
matches += 1
else:
print(f"ref_output: {ref_output.outputs[0].text}")
print(f"spec_output: {spec_output.outputs[0].text}")
# Heuristic threshold: under greedy verification, the spec-decode
# output should equal the non-spec output (modulo FP noise from the
# target's verify-path matmul running at seqlen
# num_speculative_tokens+1 vs 1). 90% leaves slack for that noise.
threshold = int(0.90 * len(ref_outputs))
print(f"match ratio: {matches}/{len(ref_outputs)}")
assert matches > threshold, (
f"match ratio {matches}/{len(ref_outputs)} <= {threshold}"
)
finally:
del lora_spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
# Under greedy verification, the spec-decode output should equal the
# non-spec output (modulo FP noise from the target's verify-path matmul
# running at seqlen num_speculative_tokens+1 vs 1). 90% leaves slack.
assert_request_outputs_match(
ref_outputs,
lora_spec_outputs,
required_matches=int(0.90 * len(ref_outputs)) + 1,
context=(
f"LoRA + {method}: target={model_name}, draft={spec_model_name}, "
f"adapter={lora_path}"
),
)