[CI] Organize speculative decoding E2E tests by coverage (#50330)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b40d859c7b
commit
fcdc7c2e9c
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test that verifies no implicit GPU-CPU synchronization occurs during
|
||||
speculative decoding generation under expected conditions.
|
||||
"""
|
||||
|
||||
import multiprocessing
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sync_tracker():
|
||||
"""
|
||||
Fixture that patches CommonAttentionMetadata.seq_lens_cpu to detect
|
||||
lazy init syncs. Prints stack traces immediately when syncs occur.
|
||||
"""
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
|
||||
# Shared counter for cross-process communication (inherited by fork)
|
||||
sync_count = multiprocessing.Value("i", 0)
|
||||
|
||||
# Save original property
|
||||
original_prop = CommonAttentionMetadata.seq_lens_cpu
|
||||
original_fget = original_prop.fget
|
||||
|
||||
# Create tracking wrapper
|
||||
def tracking_seq_lens_cpu(self):
|
||||
if self._seq_lens_cpu is None:
|
||||
# Increment counter
|
||||
with sync_count.get_lock():
|
||||
sync_count.value += 1
|
||||
count = sync_count.value
|
||||
# Print stack trace immediately (shows in subprocess output)
|
||||
print(f"\n{'=' * 60}", file=sys.stderr)
|
||||
print(f"SYNC #{count}: seq_lens_cpu lazy init triggered!", file=sys.stderr)
|
||||
print(f"{'=' * 60}", file=sys.stderr)
|
||||
traceback.print_stack(file=sys.stderr)
|
||||
print(f"{'=' * 60}\n", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
return original_fget(self)
|
||||
|
||||
# Apply patch
|
||||
CommonAttentionMetadata.seq_lens_cpu = property(tracking_seq_lens_cpu)
|
||||
|
||||
class SyncTracker:
|
||||
@property
|
||||
def count(self) -> int:
|
||||
return sync_count.value
|
||||
|
||||
def assert_no_sync(self, msg: str = ""):
|
||||
count = sync_count.value
|
||||
assert count == 0, (
|
||||
f"Unexpected GPU-CPU sync: seq_lens_cpu lazy init triggered "
|
||||
f"{count} times. See stack traces above. {msg}"
|
||||
)
|
||||
|
||||
yield SyncTracker()
|
||||
|
||||
# Restore original property
|
||||
CommonAttentionMetadata.seq_lens_cpu = original_prop
|
||||
torch._dynamo.reset()
|
||||
|
||||
|
||||
# Test configurations: (model, spec_model, method, num_spec_tokens, backend_env)
|
||||
SPEC_DECODE_CONFIGS = [
|
||||
pytest.param(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"nm-testing/Llama3_2_1B_speculator.eagle3",
|
||||
"eagle3",
|
||||
2,
|
||||
id="eagle3-llama",
|
||||
),
|
||||
pytest.param(
|
||||
"eagle618/deepseek-v3-random",
|
||||
"eagle618/eagle-deepseek-v3-random",
|
||||
"eagle",
|
||||
2,
|
||||
id="eagle-mla-deepseek",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
"mtp",
|
||||
1,
|
||||
id="mtp-qwen3_5-hybrid",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,spec_model,method,num_spec_tokens",
|
||||
SPEC_DECODE_CONFIGS,
|
||||
)
|
||||
def test_no_sync_with_spec_decode(
|
||||
sync_tracker,
|
||||
model: str,
|
||||
spec_model: str,
|
||||
method: str,
|
||||
num_spec_tokens: int,
|
||||
):
|
||||
"""
|
||||
Test that no implicit GPU-CPU sync occurs during speculative decoding
|
||||
generation.
|
||||
"""
|
||||
# Import vLLM AFTER sync_tracker fixture has applied the patch
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
# Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
|
||||
# and peaks well above the 18GB MIG slice used by one of the CI lanes.
|
||||
# This test only exercises text generation, so the vision tower is
|
||||
# never needed.
|
||||
extra_kwargs: dict = {}
|
||||
if "Qwen3.5" in model:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
|
||||
|
||||
llm = LLM(
|
||||
model=model,
|
||||
max_model_len=256,
|
||||
speculative_config={
|
||||
"method": method,
|
||||
"num_speculative_tokens": num_spec_tokens,
|
||||
"model": spec_model,
|
||||
},
|
||||
enforce_eager=True,
|
||||
async_scheduling=True,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
# Assert async scheduling is actually active before running inference.
|
||||
assert llm.llm_engine.vllm_config.scheduler_config.async_scheduling, (
|
||||
f"Expected async_scheduling=True for spec decode, got False. method={method}"
|
||||
)
|
||||
|
||||
outputs = llm.generate(
|
||||
["Hello, my name is"],
|
||||
SamplingParams(temperature=0, max_tokens=10),
|
||||
)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs[0].text) > 0
|
||||
|
||||
del llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
sync_tracker.assert_no_sync()
|
||||
@@ -0,0 +1,388 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_only, single_gpu_only
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
from ..utils import (
|
||||
Messages,
|
||||
_skip_if_insufficient_gpus_for_tp,
|
||||
compute_acceptance_len,
|
||||
compute_acceptance_rate,
|
||||
evaluate_llm_for_gsm8k,
|
||||
get_instruct_coder_messages,
|
||||
get_test_prompts,
|
||||
greedy_sampling,
|
||||
stochastic_sampling,
|
||||
)
|
||||
|
||||
|
||||
class AsyncSchedulingNotEnabledError(AssertionError):
|
||||
"""Raised when draft-model spec decode does not enable async scheduling."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArgsTest:
|
||||
target_model: str
|
||||
draft_model: str
|
||||
sampling_config: SamplingParams
|
||||
num_speculative_tokens: int
|
||||
expected_acceptance_rate: float
|
||||
expected_acceptance_len: float
|
||||
expected_gsm8k_accuracy: float = 0.0 # skip by default
|
||||
# Defaults
|
||||
enforce_eager: bool = True
|
||||
parallel_drafting: bool = False
|
||||
target_tensor_parallel_size: int = 1
|
||||
draft_tensor_parallel_size: int = 1
|
||||
max_model_len: int = 2048
|
||||
gpu_memory_utilization: float = 0.5
|
||||
dataset: str = "test_prompts"
|
||||
num_prompts: int = 100
|
||||
|
||||
|
||||
def get_messages(dataset: str, n: int) -> list[Messages]:
|
||||
if dataset == "test_prompts":
|
||||
return get_test_prompts(mm_enabled=False, num_prompts=n)
|
||||
if dataset == "likaixin/InstructCoder":
|
||||
return get_instruct_coder_messages(n=n)
|
||||
raise NotImplementedError(f"Dataset '{dataset}' not implemented")
|
||||
|
||||
|
||||
def some_high_acceptance_metrics() -> dict:
|
||||
return {
|
||||
"sampling_config": greedy_sampling(),
|
||||
"num_speculative_tokens": 3,
|
||||
"expected_acceptance_len": 3.4, # ref: 3.75
|
||||
"expected_acceptance_rate": 0.8, # ref: 0.9
|
||||
}
|
||||
|
||||
|
||||
cases = [
|
||||
# Same model for draft and target, greedy sampling.
|
||||
ArgsTest(
|
||||
target_model="Qwen/Qwen3-0.6B",
|
||||
draft_model="Qwen/Qwen3-0.6B",
|
||||
sampling_config=greedy_sampling(),
|
||||
num_speculative_tokens=3, # K
|
||||
expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1
|
||||
expected_acceptance_rate=0.98, # slight epsilon
|
||||
expected_gsm8k_accuracy=0.25, # ref: 35-40%
|
||||
),
|
||||
# Smaller draft model, stochastic sampling.
|
||||
ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
draft_model="Qwen/Qwen3-0.6B",
|
||||
sampling_config=stochastic_sampling(),
|
||||
num_speculative_tokens=3,
|
||||
expected_acceptance_len=3.4, # ref: 3.7
|
||||
expected_acceptance_rate=0.80, # ref: 0.90
|
||||
expected_gsm8k_accuracy=0.5, # ref: 60%. Note gsm8k always runs greedy sampling
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", cases)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling & engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_correctness(args: ArgsTest, enforce_eager: bool):
|
||||
args.enforce_eager = enforce_eager
|
||||
assert_draft_model_correctness(args)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_realistic_example():
|
||||
args = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
draft_model="Qwen/Qwen3-0.6B",
|
||||
dataset="likaixin/InstructCoder",
|
||||
num_speculative_tokens=3,
|
||||
sampling_config=greedy_sampling(),
|
||||
enforce_eager=False,
|
||||
expected_acceptance_len=2.6, # ref: 2.86
|
||||
expected_acceptance_rate=0.5, # ref: 0.62
|
||||
)
|
||||
assert_draft_model_correctness(args)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_parallel_drafting():
|
||||
args = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
draft_model="amd/PARD-Qwen3-0.6B",
|
||||
dataset="likaixin/InstructCoder",
|
||||
num_speculative_tokens=3,
|
||||
sampling_config=greedy_sampling(),
|
||||
parallel_drafting=True,
|
||||
enforce_eager=False,
|
||||
expected_acceptance_len=2.3, # ref: 2.52
|
||||
expected_acceptance_rate=0.4, # ref: 0.51
|
||||
)
|
||||
assert_draft_model_correctness(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"models",
|
||||
[
|
||||
# target_model, draft_model
|
||||
("Qwen/Qwen3-1.7B-FP8", "Qwen/Qwen3-0.6B"), # target quantized
|
||||
("Qwen/Qwen3-1.7B", "Qwen/Qwen3-0.6B-FP8"), # draft quantized
|
||||
],
|
||||
ids=["target_quantized", "draft_quantized"],
|
||||
)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
|
||||
tgt_model, draft_model = models
|
||||
sd_case = ArgsTest(
|
||||
target_model=tgt_model,
|
||||
draft_model=draft_model,
|
||||
**some_high_acceptance_metrics(),
|
||||
enforce_eager=enforce_eager,
|
||||
)
|
||||
assert_draft_model_correctness(sd_case)
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_tensor_parallelism():
|
||||
"""Ensure spec decode works when running with TP > 1."""
|
||||
_skip_if_insufficient_gpus_for_tp(2)
|
||||
sd_case = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
target_tensor_parallel_size=2,
|
||||
draft_model="Qwen/Qwen3-0.6B",
|
||||
draft_tensor_parallel_size=2,
|
||||
**some_high_acceptance_metrics(),
|
||||
enforce_eager=False,
|
||||
expected_gsm8k_accuracy=0.5,
|
||||
)
|
||||
assert_draft_model_correctness(sd_case)
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_engine_args_tensor_parallelism():
|
||||
"""Ensure the vllm_config for the draft model is created correctly,
|
||||
and independently of the target model (quantization, TP, etc.)"""
|
||||
_skip_if_insufficient_gpus_for_tp(2)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
|
||||
tensor_parallel_size=2,
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B", # <<< draft not quantized
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_tensor_parallel_size": 1, # <<< valid arg name
|
||||
},
|
||||
)
|
||||
target_config: VllmConfig = engine_args.create_engine_config()
|
||||
assert target_config.parallel_config.tensor_parallel_size == 2
|
||||
assert target_config.quant_config.get_name() == "fp8"
|
||||
|
||||
speculative_config = target_config.speculative_config
|
||||
draft_config: VllmConfig = replace(
|
||||
target_config,
|
||||
quant_config=None,
|
||||
parallel_config=replace(
|
||||
speculative_config.draft_parallel_config,
|
||||
rank=target_config.parallel_config.rank,
|
||||
),
|
||||
model_config=speculative_config.draft_model_config,
|
||||
)
|
||||
assert draft_config.parallel_config.tensor_parallel_size == 1
|
||||
assert draft_config.quant_config is None
|
||||
|
||||
|
||||
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
|
||||
so we can test it without instantiating a full proposer."""
|
||||
spec_cfg = vllm_config.speculative_config
|
||||
if spec_cfg.moe_backend is not None:
|
||||
return replace(
|
||||
vllm_config,
|
||||
kernel_config=replace(
|
||||
vllm_config.kernel_config,
|
||||
moe_backend=spec_cfg.moe_backend,
|
||||
),
|
||||
)
|
||||
return vllm_config
|
||||
|
||||
|
||||
def test_draft_model_moe_backend_override():
|
||||
"""When moe_backend is set in speculative_config, the draft VllmConfig
|
||||
should use it while the target keeps its own setting."""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
moe_backend="flashinfer_trtllm",
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
"moe_backend": "triton",
|
||||
},
|
||||
)
|
||||
tgt_config: VllmConfig = engine_args.create_engine_config()
|
||||
assert tgt_config.kernel_config.moe_backend == "flashinfer_trtllm"
|
||||
assert tgt_config.speculative_config.moe_backend == "triton"
|
||||
|
||||
draft_config = _apply_draft_moe_backend(tgt_config)
|
||||
assert draft_config.kernel_config.moe_backend == "triton"
|
||||
# Target config must be unaffected.
|
||||
assert tgt_config.kernel_config.moe_backend == "flashinfer_trtllm"
|
||||
|
||||
|
||||
def test_draft_model_moe_backend_inherits_target():
|
||||
"""When moe_backend is not set in speculative_config, the draft should
|
||||
inherit the target's moe_backend."""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
moe_backend="flashinfer_cutlass",
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
)
|
||||
tgt_config: VllmConfig = engine_args.create_engine_config()
|
||||
assert tgt_config.kernel_config.moe_backend == "flashinfer_cutlass"
|
||||
assert tgt_config.speculative_config.moe_backend is None
|
||||
|
||||
draft_config = _apply_draft_moe_backend(tgt_config)
|
||||
assert draft_config.kernel_config.moe_backend == "flashinfer_cutlass"
|
||||
assert draft_config is tgt_config
|
||||
|
||||
|
||||
def test_draft_model_moe_backend_default_auto():
|
||||
"""When neither target nor draft set moe_backend explicitly, both should
|
||||
default to 'auto'."""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
)
|
||||
tgt_config: VllmConfig = engine_args.create_engine_config()
|
||||
assert tgt_config.kernel_config.moe_backend == "auto"
|
||||
assert tgt_config.speculative_config.moe_backend is None
|
||||
|
||||
draft_config = _apply_draft_moe_backend(tgt_config)
|
||||
assert draft_config.kernel_config.moe_backend == "auto"
|
||||
assert draft_config is tgt_config
|
||||
|
||||
|
||||
def test_draft_model_engine_args_rejects_invalid_tp_argname():
|
||||
"""The user should pass "draft_tensor_parallel_size" rather than
|
||||
"tensor_parallel_size". We enforce this with validation."""
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
"tensor_parallel_size": 1, # <<< invalid arg name
|
||||
},
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
engine_args.create_engine_config()
|
||||
|
||||
|
||||
def assert_draft_model_correctness(args: ArgsTest):
|
||||
"""Compare the outputs using and not using speculative decoding.
|
||||
In the greedy decoding case, the outputs must match EXACTLY."""
|
||||
test_prompts: list[Messages] = get_messages(
|
||||
dataset=args.dataset, n=args.num_prompts
|
||||
)
|
||||
|
||||
spec_llm = LLM(
|
||||
model=args.target_model,
|
||||
speculative_config={
|
||||
"model": args.draft_model,
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": args.num_speculative_tokens,
|
||||
"max_model_len": args.max_model_len,
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"draft_tensor_parallel_size": args.draft_tensor_parallel_size,
|
||||
"parallel_drafting": args.parallel_drafting,
|
||||
},
|
||||
max_num_seqs=100, # limit cudagraph capture runtime
|
||||
max_model_len=args.max_model_len,
|
||||
gpu_memory_utilization=args.gpu_memory_utilization,
|
||||
tensor_parallel_size=args.target_tensor_parallel_size,
|
||||
enforce_eager=args.enforce_eager,
|
||||
disable_log_stats=False, # enables get_metrics()
|
||||
)
|
||||
|
||||
# we don't check the outputs, only check the metrics
|
||||
spec_llm.chat(test_prompts, args.sampling_config)
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate: float = compute_acceptance_rate(metrics)
|
||||
acceptance_len: float = compute_acceptance_len(metrics)
|
||||
|
||||
# Need to evaluate after getting metrics to avoid polluting the AR
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_llm, expected_accuracy_threshold=args.expected_gsm8k_accuracy
|
||||
)
|
||||
|
||||
print(
|
||||
f"spec-decode: target={args.target_model}, draft={args.draft_model}, "
|
||||
f"temperature={args.sampling_config.temperature:.2f}, "
|
||||
f"acceptance_rate={acceptance_rate:.2f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}, "
|
||||
)
|
||||
|
||||
assert acceptance_rate >= args.expected_acceptance_rate
|
||||
assert acceptance_len >= args.expected_acceptance_len
|
||||
# draft_model supports async scheduling; assert it is active by default.
|
||||
# Raise AsyncSchedulingNotEnabledError (a subclass of AssertionError) so that
|
||||
# @pytest.mark.xfail(raises=AsyncSchedulingNotEnabledError) catches only this
|
||||
# specific failure — leaving all other assertion failures (e.g. correctness or
|
||||
# acceptance-rate checks above) visible as real test failures.
|
||||
has_async = spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
del spec_llm # CLEANUP
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
if not has_async:
|
||||
raise AsyncSchedulingNotEnabledError(
|
||||
"Expected async_scheduling=True for draft_model spec decode, got False."
|
||||
" See https://github.com/vllm-project/vllm/issues/38929"
|
||||
)
|
||||
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script contains:
|
||||
1. test lora with speculative decoding for batch inference
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
LORA_TEST_PROMPT_MAP: dict[str, str] = {}
|
||||
|
||||
LORA_TEST_PROMPT_MAP["premjatin/qwen-linear-algebra-coder"] = """
|
||||
### INSTRUCTION:
|
||||
You are an AI assistant that generates Python code to solve linear
|
||||
algebra problems.
|
||||
|
||||
### PROBLEM:
|
||||
Find the eigenvalues and eigenvectors of the following 3x3 matrix:
|
||||
[[3, 2, 0],
|
||||
[2, 3, 0],
|
||||
[0, 0, 2]]
|
||||
|
||||
### OUTPUT FORMAT (STRICT):
|
||||
Numbers should be represented as integers only.
|
||||
|
||||
### PYTHON SOLUTION:
|
||||
"""
|
||||
|
||||
SEED = 42
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
|
||||
@pytest.mark.parametrize(
|
||||
"model_setup",
|
||||
[
|
||||
(
|
||||
"eagle3",
|
||||
"Qwen/Qwen3-1.7B",
|
||||
"AngelSlim/Qwen3-1.7B_eagle3",
|
||||
"premjatin/qwen-linear-algebra-coder",
|
||||
1,
|
||||
)
|
||||
],
|
||||
)
|
||||
def test_batch_inference_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str, str, int],
|
||||
):
|
||||
"""
|
||||
Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
|
||||
Should be the same and no failure when doing batch inference.
|
||||
model_setup: (method, model_name, spec_model_name, lora_path, tp_size)
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
# Disable randomness
|
||||
m.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
|
||||
set_random_seed(SEED)
|
||||
torch.backends.cudnn.benchmark = False
|
||||
torch.backends.cudnn.deterministic = True
|
||||
|
||||
method, model_name, spec_model_name, lora_path, tp_size = model_setup
|
||||
|
||||
prompts = [LORA_TEST_PROMPT_MAP[lora_path]] * 100
|
||||
lora_request = LoRARequest("adapter", 1, lora_path)
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0, top_p=1.0, top_k=-1, seed=SEED, max_tokens=128
|
||||
)
|
||||
|
||||
# without speculative decoding
|
||||
ref_llm = LLM(
|
||||
model=model_name,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=tp_size,
|
||||
max_model_len=2048,
|
||||
max_num_seqs=4,
|
||||
enable_lora=True,
|
||||
max_loras=1,
|
||||
max_cpu_loras=1,
|
||||
max_lora_rank=16,
|
||||
)
|
||||
try:
|
||||
ref_outputs = ref_llm.generate(
|
||||
prompts, sampling_params, lora_request=lora_request
|
||||
)
|
||||
finally:
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
lora_spec_llm = LLM(
|
||||
model=model_name,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=tp_size,
|
||||
speculative_config={
|
||||
"method": method,
|
||||
"model": spec_model_name,
|
||||
"num_speculative_tokens": 3,
|
||||
"max_model_len": 2048,
|
||||
},
|
||||
max_model_len=2048,
|
||||
max_num_seqs=4,
|
||||
enable_lora=True,
|
||||
max_loras=1,
|
||||
max_cpu_loras=1,
|
||||
max_lora_rank=16,
|
||||
)
|
||||
try:
|
||||
lora_spec_outputs = lora_spec_llm.generate(
|
||||
prompts, sampling_params, lora_request=lora_request
|
||||
)
|
||||
|
||||
matches = 0
|
||||
for ref_output, spec_output in zip(ref_outputs, lora_spec_outputs):
|
||||
if ref_output.outputs[0].text == spec_output.outputs[0].text:
|
||||
matches += 1
|
||||
else:
|
||||
print(f"ref_output: {ref_output.outputs[0].text}")
|
||||
print(f"spec_output: {spec_output.outputs[0].text}")
|
||||
|
||||
# Heuristic threshold: under greedy verification, the spec-decode
|
||||
# output should equal the non-spec output (modulo FP noise from the
|
||||
# target's verify-path matmul running at seqlen
|
||||
# num_speculative_tokens+1 vs 1). 90% leaves slack for that noise.
|
||||
threshold = int(0.90 * len(ref_outputs))
|
||||
print(f"match ratio: {matches}/{len(ref_outputs)}")
|
||||
assert matches > threshold, (
|
||||
f"match ratio {matches}/{len(ref_outputs)} <= {threshold}"
|
||||
)
|
||||
finally:
|
||||
del lora_spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
Reference in New Issue
Block a user