[CI] Solidify speculative decoding E2E coverage (#50713)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
48bada6ea4
commit
1a1727330a
@@ -102,14 +102,15 @@ def test_no_sync_with_spec_decode(
|
||||
spec_model: str,
|
||||
method: str,
|
||||
num_spec_tokens: int,
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Test that no implicit GPU-CPU sync occurs during speculative decoding
|
||||
generation.
|
||||
"""
|
||||
# Import vLLM AFTER sync_tracker fixture has applied the patch
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
|
||||
# Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
|
||||
# and peaks well above the 18GB MIG slice used by one of the CI lanes.
|
||||
@@ -119,8 +120,10 @@ def test_no_sync_with_spec_decode(
|
||||
if "Qwen3.5" in model:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
|
||||
|
||||
llm = LLM(
|
||||
model=model,
|
||||
with vllm_runner(
|
||||
model,
|
||||
block_size=None,
|
||||
trust_remote_code=False,
|
||||
max_model_len=256,
|
||||
speculative_config={
|
||||
"method": method,
|
||||
@@ -129,24 +132,31 @@ def test_no_sync_with_spec_decode(
|
||||
},
|
||||
enforce_eager=True,
|
||||
async_scheduling=True,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
|
||||
# Assert async scheduling is actually active before running inference.
|
||||
assert llm.llm_engine.vllm_config.scheduler_config.async_scheduling, (
|
||||
f"Expected async_scheduling=True for spec decode, got False. method={method}"
|
||||
)
|
||||
# Assert async scheduling is actually active before running inference.
|
||||
assert llm.llm_engine.vllm_config.scheduler_config.async_scheduling, (
|
||||
f"Expected async_scheduling=True for spec decode, got False. "
|
||||
f"method={method}, target={model}, draft={spec_model}"
|
||||
)
|
||||
|
||||
outputs = llm.generate(
|
||||
["Hello, my name is"],
|
||||
SamplingParams(temperature=0, max_tokens=10),
|
||||
)
|
||||
outputs = llm.generate(
|
||||
["Hello, my name is"],
|
||||
SamplingParams(temperature=0, max_tokens=10),
|
||||
)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs[0].text) > 0
|
||||
|
||||
del llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
assert len(outputs) == 1, (
|
||||
f"{method} target={model}: expected one request output, got {len(outputs)}"
|
||||
)
|
||||
assert outputs[0].outputs, (
|
||||
f"{method} target={model}: request output has no completion candidates"
|
||||
)
|
||||
assert outputs[0].outputs[0].text, (
|
||||
f"{method} target={model}: generated completion text is empty"
|
||||
)
|
||||
|
||||
sync_tracker.assert_no_sync()
|
||||
|
||||
@@ -4,17 +4,16 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_only, single_gpu_only
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig, VllmConfig, replace
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
Messages,
|
||||
_skip_if_insufficient_gpus_for_tp,
|
||||
compute_acceptance_len,
|
||||
compute_acceptance_rate,
|
||||
evaluate_llm_for_gsm8k,
|
||||
@@ -42,7 +41,8 @@ class ArgsTest:
|
||||
max_model_len: int = 2048
|
||||
gpu_memory_utilization: float = 0.5
|
||||
dataset: str = "test_prompts"
|
||||
num_prompts: int = 100
|
||||
# Doubling 100 to 200 reduces sampling standard error by about 29%.
|
||||
num_prompts: int = 200
|
||||
|
||||
|
||||
def get_messages(dataset: str, n: int) -> list[Messages]:
|
||||
@@ -89,13 +89,13 @@ cases = [
|
||||
@pytest.mark.parametrize("args", cases)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@single_gpu_only
|
||||
def test_draft_model_correctness(args: ArgsTest, enforce_eager: bool):
|
||||
def test_draft_model_correctness(args: ArgsTest, enforce_eager: bool, vllm_runner):
|
||||
args.enforce_eager = enforce_eager
|
||||
assert_draft_model_correctness(args)
|
||||
assert_draft_model_correctness(args, vllm_runner)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_draft_model_realistic_example():
|
||||
def test_draft_model_realistic_example(vllm_runner):
|
||||
args = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
draft_model="Qwen/Qwen3-0.6B",
|
||||
@@ -106,11 +106,11 @@ def test_draft_model_realistic_example():
|
||||
expected_acceptance_len=2.6, # ref: 2.86
|
||||
expected_acceptance_rate=0.5, # ref: 0.62
|
||||
)
|
||||
assert_draft_model_correctness(args)
|
||||
assert_draft_model_correctness(args, vllm_runner)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_draft_model_parallel_drafting():
|
||||
def test_draft_model_parallel_drafting(vllm_runner):
|
||||
args = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
draft_model="amd/PARD-Qwen3-0.6B",
|
||||
@@ -122,7 +122,7 @@ def test_draft_model_parallel_drafting():
|
||||
expected_acceptance_len=2.3, # ref: 2.52
|
||||
expected_acceptance_rate=0.4, # ref: 0.51
|
||||
)
|
||||
assert_draft_model_correctness(args)
|
||||
assert_draft_model_correctness(args, vllm_runner)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -136,7 +136,9 @@ def test_draft_model_parallel_drafting():
|
||||
)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@single_gpu_only
|
||||
def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
|
||||
def test_draft_model_quantization(
|
||||
models: tuple[str, str], enforce_eager: bool, vllm_runner
|
||||
):
|
||||
tgt_model, draft_model = models
|
||||
sd_case = ArgsTest(
|
||||
target_model=tgt_model,
|
||||
@@ -144,13 +146,12 @@ def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
|
||||
**some_high_acceptance_metrics(),
|
||||
enforce_eager=enforce_eager,
|
||||
)
|
||||
assert_draft_model_correctness(sd_case)
|
||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_tensor_parallelism():
|
||||
def test_draft_model_tensor_parallelism(vllm_runner):
|
||||
"""Ensure spec decode works when running with TP > 1."""
|
||||
_skip_if_insufficient_gpus_for_tp(2)
|
||||
sd_case = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
target_tensor_parallel_size=2,
|
||||
@@ -160,15 +161,13 @@ def test_draft_model_tensor_parallelism():
|
||||
enforce_eager=False,
|
||||
expected_gsm8k_accuracy=0.5,
|
||||
)
|
||||
assert_draft_model_correctness(sd_case)
|
||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_engine_args_tensor_parallelism():
|
||||
"""Ensure the vllm_config for the draft model is created correctly,
|
||||
and independently of the target model (quantization, TP, etc.)"""
|
||||
_skip_if_insufficient_gpus_for_tp(2)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
|
||||
tensor_parallel_size=2,
|
||||
@@ -212,13 +211,24 @@ def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
return vllm_config
|
||||
|
||||
|
||||
def test_draft_model_moe_backend_override():
|
||||
def _platform_moe_backend(monkeypatch: pytest.MonkeyPatch) -> MoEBackend:
|
||||
if current_platform.is_rocm():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
return "aiter"
|
||||
if current_platform.is_cuda():
|
||||
return "flashinfer_trtllm"
|
||||
return "auto"
|
||||
|
||||
|
||||
def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When moe_backend is set in speculative_config, the draft VllmConfig
|
||||
should use it while the target keeps its own setting."""
|
||||
target_moe_backend = _platform_moe_backend(monkeypatch)
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
moe_backend="flashinfer_trtllm",
|
||||
moe_backend=target_moe_backend,
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
@@ -227,22 +237,23 @@ def test_draft_model_moe_backend_override():
|
||||
},
|
||||
)
|
||||
tgt_config: VllmConfig = engine_args.create_engine_config()
|
||||
assert tgt_config.kernel_config.moe_backend == "flashinfer_trtllm"
|
||||
assert tgt_config.kernel_config.moe_backend == target_moe_backend
|
||||
assert tgt_config.speculative_config.moe_backend == "triton"
|
||||
|
||||
draft_config = _apply_draft_moe_backend(tgt_config)
|
||||
assert draft_config.kernel_config.moe_backend == "triton"
|
||||
# Target config must be unaffected.
|
||||
assert tgt_config.kernel_config.moe_backend == "flashinfer_trtllm"
|
||||
assert tgt_config.kernel_config.moe_backend == target_moe_backend
|
||||
|
||||
|
||||
def test_draft_model_moe_backend_inherits_target():
|
||||
def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When moe_backend is not set in speculative_config, the draft should
|
||||
inherit the target's moe_backend."""
|
||||
target_moe_backend = _platform_moe_backend(monkeypatch)
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
moe_backend="flashinfer_cutlass",
|
||||
moe_backend=target_moe_backend,
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
@@ -250,11 +261,11 @@ def test_draft_model_moe_backend_inherits_target():
|
||||
},
|
||||
)
|
||||
tgt_config: VllmConfig = engine_args.create_engine_config()
|
||||
assert tgt_config.kernel_config.moe_backend == "flashinfer_cutlass"
|
||||
assert tgt_config.kernel_config.moe_backend == target_moe_backend
|
||||
assert tgt_config.speculative_config.moe_backend is None
|
||||
|
||||
draft_config = _apply_draft_moe_backend(tgt_config)
|
||||
assert draft_config.kernel_config.moe_backend == "flashinfer_cutlass"
|
||||
assert draft_config.kernel_config.moe_backend == target_moe_backend
|
||||
assert draft_config is tgt_config
|
||||
|
||||
|
||||
@@ -293,19 +304,23 @@ def test_draft_model_engine_args_rejects_invalid_tp_argname():
|
||||
"tensor_parallel_size": 1, # <<< invalid arg name
|
||||
},
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
with pytest.raises(ValueError, match="draft_tensor_parallel_size"):
|
||||
engine_args.create_engine_config()
|
||||
|
||||
|
||||
def assert_draft_model_correctness(args: ArgsTest):
|
||||
def assert_draft_model_correctness(args: ArgsTest, vllm_runner):
|
||||
"""Compare the outputs using and not using speculative decoding.
|
||||
In the greedy decoding case, the outputs must match EXACTLY."""
|
||||
test_prompts: list[Messages] = get_messages(
|
||||
dataset=args.dataset, n=args.num_prompts
|
||||
)
|
||||
|
||||
spec_llm = LLM(
|
||||
model=args.target_model,
|
||||
with vllm_runner(
|
||||
args.target_model,
|
||||
block_size=None,
|
||||
trust_remote_code=False,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
speculative_config={
|
||||
"model": args.draft_model,
|
||||
"method": "draft_model",
|
||||
@@ -321,30 +336,41 @@ def assert_draft_model_correctness(args: ArgsTest):
|
||||
tensor_parallel_size=args.target_tensor_parallel_size,
|
||||
enforce_eager=args.enforce_eager,
|
||||
disable_log_stats=False, # enables get_metrics()
|
||||
)
|
||||
) as spec_runner:
|
||||
spec_llm = spec_runner.llm
|
||||
|
||||
# we don't check the outputs, only check the metrics
|
||||
spec_llm.chat(test_prompts, args.sampling_config)
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate: float = compute_acceptance_rate(metrics)
|
||||
acceptance_len: float = compute_acceptance_len(metrics)
|
||||
# we don't check the outputs, only check the metrics
|
||||
spec_llm.chat(test_prompts, args.sampling_config)
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate: float = compute_acceptance_rate(metrics)
|
||||
acceptance_len: float = compute_acceptance_len(metrics)
|
||||
|
||||
# Need to evaluate after getting metrics to avoid polluting the AR
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_llm, expected_accuracy_threshold=args.expected_gsm8k_accuracy
|
||||
)
|
||||
# Need to evaluate after getting metrics to avoid polluting the AR
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_llm, expected_accuracy_threshold=args.expected_gsm8k_accuracy
|
||||
)
|
||||
|
||||
print(
|
||||
f"spec-decode: target={args.target_model}, draft={args.draft_model}, "
|
||||
f"temperature={args.sampling_config.temperature:.2f}, "
|
||||
f"acceptance_rate={acceptance_rate:.2f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}, "
|
||||
)
|
||||
print(
|
||||
f"spec-decode: target={args.target_model}, draft={args.draft_model}, "
|
||||
f"temperature={args.sampling_config.temperature:.2f}, "
|
||||
f"acceptance_rate={acceptance_rate:.2f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}, "
|
||||
)
|
||||
|
||||
context = (
|
||||
f"draft_model target={args.target_model}, draft={args.draft_model}, "
|
||||
f"eager={args.enforce_eager}, "
|
||||
f"acceptance_rate={acceptance_rate:.3f}, "
|
||||
f"acceptance_len={acceptance_len:.3f}"
|
||||
)
|
||||
assert acceptance_rate >= args.expected_acceptance_rate, (
|
||||
f"{context}; expected acceptance_rate >= "
|
||||
f"{args.expected_acceptance_rate:.3f}"
|
||||
)
|
||||
assert acceptance_len >= args.expected_acceptance_len, (
|
||||
f"{context}; expected acceptance_len >= {args.expected_acceptance_len:.3f}"
|
||||
)
|
||||
# draft_model supports async scheduling; assert it is active by default.
|
||||
has_async = spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
|
||||
assert acceptance_rate >= args.expected_acceptance_rate
|
||||
assert acceptance_len >= args.expected_acceptance_len
|
||||
has_async = spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
del spec_llm # CLEANUP
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
assert has_async, "Expected async_scheduling=True for draft_model spec decode"
|
||||
|
||||
@@ -8,12 +8,14 @@ This script contains:
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
from ..utils import assert_request_outputs_match
|
||||
|
||||
LORA_TEST_PROMPT_MAP: dict[str, str] = {}
|
||||
|
||||
LORA_TEST_PROMPT_MAP["premjatin/qwen-linear-algebra-coder"] = """
|
||||
@@ -36,7 +38,6 @@ Numbers should be represented as integers only.
|
||||
SEED = 42
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
|
||||
@pytest.mark.parametrize(
|
||||
"model_setup",
|
||||
[
|
||||
@@ -49,9 +50,13 @@ SEED = 42
|
||||
)
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
|
||||
)
|
||||
def test_batch_inference_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str, str, int],
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
|
||||
@@ -60,10 +65,11 @@ def test_batch_inference_correctness(
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
# Disable randomness
|
||||
m.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
|
||||
if current_platform.is_cuda():
|
||||
m.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
|
||||
set_random_seed(SEED)
|
||||
torch.backends.cudnn.benchmark = False
|
||||
torch.backends.cudnn.deterministic = True
|
||||
m.setattr(torch.backends.cudnn, "benchmark", False)
|
||||
m.setattr(torch.backends.cudnn, "deterministic", True)
|
||||
|
||||
method, model_name, spec_model_name, lora_path, tp_size = model_setup
|
||||
|
||||
@@ -74,8 +80,9 @@ def test_batch_inference_correctness(
|
||||
)
|
||||
|
||||
# without speculative decoding
|
||||
ref_llm = LLM(
|
||||
model=model_name,
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=tp_size,
|
||||
max_model_len=2048,
|
||||
@@ -84,18 +91,16 @@ def test_batch_inference_correctness(
|
||||
max_loras=1,
|
||||
max_cpu_loras=1,
|
||||
max_lora_rank=16,
|
||||
)
|
||||
try:
|
||||
ref_outputs = ref_llm.generate(
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
) as ref_runner:
|
||||
ref_outputs = ref_runner.llm.generate(
|
||||
prompts, sampling_params, lora_request=lora_request
|
||||
)
|
||||
finally:
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
lora_spec_llm = LLM(
|
||||
model=model_name,
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=tp_size,
|
||||
speculative_config={
|
||||
@@ -110,30 +115,22 @@ def test_batch_inference_correctness(
|
||||
max_loras=1,
|
||||
max_cpu_loras=1,
|
||||
max_lora_rank=16,
|
||||
)
|
||||
try:
|
||||
lora_spec_outputs = lora_spec_llm.generate(
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
) as spec_runner:
|
||||
lora_spec_outputs = spec_runner.llm.generate(
|
||||
prompts, sampling_params, lora_request=lora_request
|
||||
)
|
||||
|
||||
matches = 0
|
||||
for ref_output, spec_output in zip(ref_outputs, lora_spec_outputs):
|
||||
if ref_output.outputs[0].text == spec_output.outputs[0].text:
|
||||
matches += 1
|
||||
else:
|
||||
print(f"ref_output: {ref_output.outputs[0].text}")
|
||||
print(f"spec_output: {spec_output.outputs[0].text}")
|
||||
|
||||
# Heuristic threshold: under greedy verification, the spec-decode
|
||||
# output should equal the non-spec output (modulo FP noise from the
|
||||
# target's verify-path matmul running at seqlen
|
||||
# num_speculative_tokens+1 vs 1). 90% leaves slack for that noise.
|
||||
threshold = int(0.90 * len(ref_outputs))
|
||||
print(f"match ratio: {matches}/{len(ref_outputs)}")
|
||||
assert matches > threshold, (
|
||||
f"match ratio {matches}/{len(ref_outputs)} <= {threshold}"
|
||||
)
|
||||
finally:
|
||||
del lora_spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
# Under greedy verification, the spec-decode output should equal the
|
||||
# non-spec output (modulo FP noise from the target's verify-path matmul
|
||||
# running at seqlen num_speculative_tokens+1 vs 1). 90% leaves slack.
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
lora_spec_outputs,
|
||||
required_matches=int(0.90 * len(ref_outputs)) + 1,
|
||||
context=(
|
||||
f"LoRA + {method}: target={model_name}, draft={spec_model_name}, "
|
||||
f"adapter={lora_path}"
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user