[CI] Organize speculative decoding E2E tests by coverage (#50330)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Michael Goin
2026-07-31 18:05:42 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b40d859c7b
commit fcdc7c2e9c
39 changed files with 1941 additions and 1863 deletions
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import pytest
import torch
from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k_offline
from tests.utils import single_gpu_only
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from ...utils import compute_acceptance_len
from ..utils import run_acceptance_length_eval
@dataclass(frozen=True)
class DFlashCorrectnessConfig:
model: str
draft_model: str
expected_accuracy: float
expected_acceptance_len: float
num_speculative_tokens: int = 16
max_model_len: int = 4096
max_num_seqs: int = 128
num_questions: int = 1319
use_chat_completions: bool = False
enforce_eager: bool = False
disable_flashinfer_sampler: bool = False
QWEN3_DFLASH = DFlashCorrectnessConfig(
model="Qwen/Qwen3-8B",
draft_model="z-lab/Qwen3-8B-DFlash-b16",
expected_accuracy=0.8,
expected_acceptance_len=3.5,
)
LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig(
model="poolside/Laguna-XS-2.1-NVFP4",
draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4",
expected_accuracy=0.7, # Standard GSM8K sanity floor.
expected_acceptance_len=3.55 * 0.9,
num_speculative_tokens=15,
max_model_len=8192,
max_num_seqs=32,
num_questions=200,
use_chat_completions=True,
enforce_eager=True,
disable_flashinfer_sampler=True,
)
@pytest.mark.parametrize("use_mrv2", [False, True])
def test_dflash_reference_acceptance_lengths(
monkeypatch: pytest.MonkeyPatch,
use_mrv2: bool,
):
run_acceptance_length_eval(
monkeypatch,
spec_config={
"model": QWEN3_DFLASH.model,
"trust_remote_code": True,
"speculative_config": {
"method": "dflash",
"model": QWEN3_DFLASH.draft_model,
"num_speculative_tokens": QWEN3_DFLASH.num_speculative_tokens,
"max_model_len": 32768,
},
"max_model_len": 32768,
"max_num_seqs": 128,
"gpu_memory_utilization": 0.85,
"enforce_eager": False,
"disable_log_stats": False,
},
# Table 1 in https://arxiv.org/pdf/2602.06036.
expected_acceptance_lengths={
"mt-bench": 4.24,
"humaneval": 6.50,
"gsm8k": 6.54 * 0.975,
},
chat_template_kwargs={"enable_thinking": False},
use_mrv2=use_mrv2,
)
@single_gpu_only
@pytest.mark.parametrize(
("config", "use_mrv2"),
[
pytest.param(
QWEN3_DFLASH,
False,
id="qwen3-mrv1",
),
pytest.param(
QWEN3_DFLASH,
True,
id="qwen3-mrv2",
),
pytest.param(
LAGUNA_DFLASH_NVFP4,
True,
id="laguna-nvfp4-mrv2",
),
],
)
def test_dflash_correctness(
monkeypatch: pytest.MonkeyPatch,
config: DFlashCorrectnessConfig,
use_mrv2: bool,
):
"""Guard GSM8K accuracy and batched acceptance length for DFlash models."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
if config.disable_flashinfer_sampler:
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
spec_llm = LLM(
model=config.model,
trust_remote_code=True,
speculative_config={
"method": "dflash",
"model": config.draft_model,
"num_speculative_tokens": config.num_speculative_tokens,
"max_model_len": config.max_model_len,
},
max_model_len=config.max_model_len,
max_num_seqs=config.max_num_seqs,
gpu_memory_utilization=0.85,
enforce_eager=config.enforce_eager,
disable_log_stats=False,
)
results = evaluate_gsm8k_offline(
spec_llm,
num_questions=config.num_questions,
use_chat_completions=config.use_chat_completions,
)
accuracy = results["accuracy"]
acceptance_len = compute_acceptance_len(spec_llm.get_metrics())
print(
f"{config.model}: GSM8K accuracy={accuracy:.3f}, "
f"acceptance_len={acceptance_len:.2f}"
)
assert accuracy >= config.expected_accuracy
assert acceptance_len >= config.expected_acceptance_len
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k_offline
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from ...utils import compute_acceptance_len, compute_acceptance_rate
def test_gemma4_dspark_correctness_and_acceptance_rate(
monkeypatch: pytest.MonkeyPatch,
):
"""
E2E test for Gemma4 DSpark speculative decoding: acceptance rate/length
regression coverage plus GSM8K correctness, at temperature=1.0 to exercise
the probabilistic draft-sampling/rejection-sampling path (not just greedy).
gemma-4-12B is instruct-tuned, so GSM8K is run through the chat template
(use_chat_completions=True; raw few-shot completion collapses to a few
percent). Reference: measured over 5 runs of 200 GSM8K questions at
temperature=1.0 (prefix caching disabled):
accuracy: min=0.900 max=0.955 mean=0.937
acceptance_rate: min=0.578 max=0.595 mean=0.588
acceptance_len: min=5.044 max=5.167 mean=5.116
Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling
"""
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
spec_llm = LLM(
model="RedHatAI/gemma-4-12B-it-NVFP4",
trust_remote_code=True,
speculative_config={
"method": "dspark",
"model": "deepseek-ai/dspark_gemma4_12b_block7",
"num_speculative_tokens": 7,
"draft_sample_method": "probabilistic",
},
max_model_len=4096,
max_num_seqs=32,
gpu_memory_utilization=0.85,
enforce_eager=True,
enable_prefix_caching=False,
disable_log_stats=False,
)
try:
results = evaluate_gsm8k_offline(
spec_llm, num_questions=200, temperature=1.0, use_chat_completions=True
)
gsm8k_accuracy = results["accuracy"]
metrics = spec_llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
acceptance_len = compute_acceptance_len(metrics)
print(
f"Gemma4 DSpark acceptance_rate={acceptance_rate:.2f}, "
f"acceptance_len={acceptance_len:.2f}, "
f"gsm8k_accuracy={gsm8k_accuracy:.3f}"
)
assert acceptance_rate >= 0.58 * 0.9
assert acceptance_len >= 5.0 * 0.9
assert gsm8k_accuracy >= 0.92 * 0.9
finally:
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@pytest.fixture
def dspark_config():
target_model = "Qwen/Qwen3-4B-FP8"
draft_model = "deepseek-ai/dspark_qwen3_4b_block7"
return dict(
model=target_model,
trust_remote_code=True,
speculative_config={
"method": "dspark",
"model": draft_model,
"num_speculative_tokens": 7,
"attention_backend": "FLASH_ATTN",
"draft_sample_method": "probabilistic",
},
max_model_len=4096,
disable_log_stats=False,
)
def test_dspark_correctness_and_acceptance_rate(dspark_config):
"""
E2E test for DSpark speculative decoding: acceptance rate/length
regression coverage plus GSM8K correctness, at temperature=1.0 to
exercise the probabilistic draft-sampling/rejection-sampling path
(not just greedy).
Uses Qwen/Qwen3-4B-FP8 as target with the dspark_qwen3_4b_block7 draft
model. Reference: measured over 12 runs of the full GSM8K set at
temperature=1.0 (prefix caching disabled to avoid cross-run reuse):
accuracy: min=0.782 max=0.814 mean=0.801
acceptance_rate: min=0.418 max=0.434 mean=0.428
acceptance_len: min=3.928 max=4.037 mean=3.994
Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling
"""
spec_llm = LLM(**dspark_config)
results = evaluate_gsm8k_offline(spec_llm, temperature=1.0)
gsm8k_accuracy = results["accuracy"]
metrics = spec_llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
acceptance_len = compute_acceptance_len(metrics)
print(
f"DSpark acceptance_rate={acceptance_rate:.2f}, "
f"acceptance_len={acceptance_len:.2f}, "
f"gsm8k_accuracy={gsm8k_accuracy:.3f}"
)
assert acceptance_rate >= 0.428 * 0.9
assert acceptance_len >= 3.994 * 0.9
assert gsm8k_accuracy >= 0.801 * 0.9
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from ...utils import compute_acceptance_rate
def test_medusa_acceptance_rate(
sampling_config: SamplingParams,
):
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
speculation path actually accepts tokens — unlike test_medusa_correctness,
which uses a random head and only validates output correctness.
"""
target_model = "lmsys/vicuna-7b-v1.3"
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
spec_llm = LLM(
model=target_model,
speculative_config={
"method": "medusa",
"model": medusa_model,
"num_speculative_tokens": 3,
},
max_model_len=1024,
enforce_eager=True,
disable_log_stats=False,
)
spec_llm.generate(prompts, sampling_config)
metrics = spec_llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
min_acceptance_rate = 0.198
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
# Regression guard at 90% of the measured baseline.
assert acceptance_rate >= min_acceptance_rate, (
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
)
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from ..utils import run_acceptance_length_eval
@pytest.mark.parametrize("use_mrv2", [False, True])
def test_gemma4_mtp_acceptance_lengths(
monkeypatch: pytest.MonkeyPatch,
use_mrv2: bool,
):
run_acceptance_length_eval(
monkeypatch,
spec_config={
"model": "google/gemma-4-E4B-it",
"trust_remote_code": True,
"speculative_config": {
"method": "mtp",
"model": "google/gemma-4-E4B-it-assistant",
"num_speculative_tokens": 2,
"max_model_len": 32768,
},
"max_model_len": 32768,
"limit_mm_per_prompt": {"image": 0, "audio": 0},
"disable_log_stats": False,
},
expected_acceptance_lengths={
"mt-bench": 2.28,
"humaneval": 2.68,
"gsm8k": 2.67 * 0.975,
},
chat_template_kwargs={},
use_mrv2=use_mrv2,
)
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from tests.utils import single_gpu_only
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from ...utils import compute_acceptance_len, get_test_prompts
@single_gpu_only
def test_synthetic_acceptance_rate():
"""Verify that synthetic rejection sampling produces an acceptance
length close to the requested mean acceptance length."""
num_spec_tokens = 3
expected_acceptance_len = 1.875
tolerance = 0.15
spec_llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
trust_remote_code=True,
speculative_config={
"method": "eagle3",
"model": "nm-testing/Llama3_2_1B_speculator.eagle3",
"num_speculative_tokens": num_spec_tokens,
"max_model_len": 2048,
"rejection_sample_method": "synthetic",
"synthetic_acceptance_length": expected_acceptance_len,
},
max_model_len=2048,
enforce_eager=True,
disable_log_stats=False,
)
test_prompts = get_test_prompts(mm_enabled=False, num_prompts=50)
spec_llm.chat(
test_prompts,
SamplingParams(temperature=0, max_tokens=64, ignore_eos=True),
)
metrics = spec_llm.get_metrics()
acceptance_len = compute_acceptance_len(metrics)
print(
f"Synthetic acceptance length: {acceptance_len:.3f}"
f" (expected={expected_acceptance_len:.3f},"
f" tolerance=±{tolerance})"
)
assert abs(acceptance_len - expected_acceptance_len) <= tolerance, (
f"Synthetic acceptance length {acceptance_len:.3f} is not within"
f" ±{tolerance} of expected {expected_acceptance_len:.3f}"
)
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@@ -0,0 +1,105 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
import torch
from tqdm import tqdm
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from ..utils import compute_acceptance_len
def load_and_process_dataset(data_name: str):
from datasets import load_dataset
if data_name == "gsm8k":
dataset = load_dataset("openai/gsm8k", "main", split="test")
prompt_fmt = (
"{question}\nPlease reason step by step,"
" and put your final answer within \\boxed{{}}."
)
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
elif data_name == "mt-bench":
dataset = load_dataset("HuggingFaceH4/mt_bench_prompts", split="train")
dataset = dataset.map(lambda x: {"turns": x["prompt"]})
elif data_name == "humaneval":
dataset = load_dataset("openai/openai_humaneval", split="test")
prompt_fmt = (
"Write a solution to the following problem and make sure"
" that it passes the tests:\n```python\n{prompt}\n```"
)
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
return dataset
def run_acceptance_length_eval(
monkeypatch: pytest.MonkeyPatch,
spec_config: dict[str, Any],
expected_acceptance_lengths: dict[str, float],
chat_template_kwargs: dict[str, Any],
use_mrv2: bool,
):
"""
E2E acceptance-rate validation for speculative decoding.
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
through the spec decode engine and asserts the mean acceptance length
stays within tolerance of the reference figure for each dataset.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
spec_llm = LLM(**spec_config)
max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k
tokenizer = spec_llm.get_tokenizer()
for dataset_name, expected_len in expected_acceptance_lengths.items():
dataset = load_and_process_dataset(dataset_name)
prev_metrics = None
acceptance_lengths = []
for i in tqdm(
range(min(max_prompts_per_dataset, len(dataset))),
desc=f"Processing {dataset_name}",
):
user_content = dataset[i]["turns"][0]
prompt_text = tokenizer.apply_chat_template(
[{"role": "user", "content": user_content}],
tokenize=False,
add_generation_prompt=True,
**chat_template_kwargs,
)
# Greedy (temp=0) so acceptance length is deterministic and comparable
# across runs.
spec_llm.generate(
[prompt_text],
SamplingParams(temperature=0, max_tokens=2048),
use_tqdm=False,
)
current_metrics = spec_llm.get_metrics()
acceptance_len = compute_acceptance_len(current_metrics, prev_metrics)
prev_metrics = current_metrics
acceptance_lengths.append(acceptance_len)
mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths)
# Fairly tight tolerance of 95% against the reference figures,
# watching for regressions. Can be relaxed if test is flaky but be sure to
# check for genuine issues such as #40727.
expected_len = expected_len * 0.95
print(
f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
f" (expected at least {expected_len:.2f})"
)
assert mean_acceptance_length >= expected_len, (
f"acceptance_len for {dataset_name} is below expected threshold: "
f"{mean_acceptance_length:.2f} < {expected_len:.2f}"
)
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()