[Tests][Spec Decode] Add gemma4 MTP acceptance rates test (#47920)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-27 16:15:54 -07:00
committed by GitHub
parent 1e34a13539
commit 272abd5f48
4 changed files with 120 additions and 48 deletions
+12
View File
@@ -188,3 +188,15 @@ steps:
- tests/v1/e2e/spec_decode/test_mtp_parallel_load.py
commands:
- pytest -v -s v1/e2e/spec_decode/test_mtp_parallel_load.py
- label: Spec Decode Acceptance Rates Nightly
key: spec-decode-acceptance-rates-nightly
timeout_in_minutes: 60
device: h200_35gb
optional: true
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "acceptance_rates"
+73 -42
View File
@@ -1393,51 +1393,81 @@ def load_and_process_dataset(data_name: str):
return dataset
@pytest.fixture
def dflash_config():
target_model = "Qwen/Qwen3-8B"
draft_model = "z-lab/Qwen3-8B-DFlash-b16"
return dict(
model=target_model,
trust_remote_code=True,
speculative_config={
"method": "dflash",
"model": draft_model,
"num_speculative_tokens": 16,
"max_model_len": 32768,
},
max_model_len=32768,
max_num_seqs=128,
gpu_memory_utilization=0.85,
enforce_eager=False,
disable_log_stats=False,
)
@pytest.mark.parametrize(
["spec_config", "expected_acceptance_lengths", "chat_template_kwargs"],
[
pytest.param(
dict(
model="Qwen/Qwen3-8B",
trust_remote_code=True,
speculative_config={
"method": "dflash",
"model": "z-lab/Qwen3-8B-DFlash-b16",
"num_speculative_tokens": 16,
"max_model_len": 32768,
},
max_model_len=32768,
max_num_seqs=128,
gpu_memory_utilization=0.85,
enforce_eager=False,
disable_log_stats=False,
),
# All scores from Table 1 in https://arxiv.org/pdf/2602.06036
{
"mt-bench": 4.24,
"humaneval": 6.50,
# runs with a subset of prompts so extra wide tol here
"gsm8k": 6.54 * 0.975,
},
{"enable_thinking": False},
id="dflash",
),
pytest.param(
dict(
model="google/gemma-4-E4B-it",
trust_remote_code=True,
speculative_config={
"method": "mtp",
"model": "google/gemma-4-E4B-it-assistant",
"num_speculative_tokens": 2,
"max_model_len": 32768,
},
max_model_len=32768,
# Skip multimodal profiling; this is a text-only eval.
limit_mm_per_prompt={"image": 0, "audio": 0},
disable_log_stats=False,
),
{
"mt-bench": 2.28,
"humaneval": 2.68,
# runs with a subset of prompts so extra wide tol here
"gsm8k": 2.67 * 0.975,
},
{},
id="gemma4",
),
],
)
@pytest.mark.parametrize("use_mrv2", [False, True])
def test_dflash_acceptance_rates(
monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config
def test_acceptance_rates(
monkeypatch: pytest.MonkeyPatch,
spec_config: dict[str, Any],
expected_acceptance_lengths: dict[str, float],
chat_template_kwargs: dict[str, Any],
use_mrv2: bool,
):
"""
E2E test for DFlash (block diffusion) speculative decoding.
Runs acceptance rate validation on GSM8k, MT-Bench, and HumanEval
comparing against baseline results from the paper (Table 1).
See https://github.com/z-lab/dflash/blob/main/benchmark_sglang.py for methodology.
E2E acceptance-rate validation for speculative decoding.
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
through the spec decode engine and asserts the mean acceptance length
stays within tolerance of the reference figure for each dataset.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
spec_llm = LLM(**dflash_config)
spec_llm = LLM(**spec_config)
max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k
# All scores from Table 1 in https://arxiv.org/pdf/2602.06036
expected_acceptance_lengths = {
"mt-bench": 4.24,
"humaneval": 6.50,
"gsm8k": 6.54 * 0.975, # runs with a subset of prompts so extra wide tol here
}
tokenizer = spec_llm.get_tokenizer()
for dataset_name, expected_len in expected_acceptance_lengths.items():
dataset = load_and_process_dataset(dataset_name)
@@ -1452,10 +1482,11 @@ def test_dflash_acceptance_rates(
[{"role": "user", "content": user_content}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
**chat_template_kwargs,
)
# Temp=0, MaxTokens=2048 from the paper
# Greedy (temp=0) so acceptance length is deterministic and comparable
# across runs.
spec_llm.generate(
[prompt_text],
SamplingParams(temperature=0, max_tokens=2048),
@@ -1467,17 +1498,17 @@ def test_dflash_acceptance_rates(
acceptance_lengths.append(acceptance_len)
mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths)
# Fairly tight tolerance of 95% against the paper's figures,
# Fairly tight tolerance of 95% against the reference figures,
# watching for regressions. Can be relaxed if test is flaky but be sure to
# check for genuine issues such as #40727.
expected_len = expected_len * 0.95
print(
f"DFlash acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
f" (expected at least {expected_len:.2f})"
)
assert mean_acceptance_length >= expected_len, (
f"DFlash acceptance_len for {dataset_name} is below expected threshold:"
f"acceptance_len for {dataset_name} is below expected threshold: "
f"{mean_acceptance_length:.2f} < {expected_len:.2f}"
)
+24
View File
@@ -14,6 +14,7 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_layers_from_vllm_config, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.v1.attention.backend import CommonAttentionMetadata
@@ -165,6 +166,29 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
)
return base
def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None:
"""Gemma4 MTP requires dim-mismatched embedding sharing.
The draft checkpoint's embed_tokens is a draft-dim placeholder
(tied to lm_head so load_weights populates both); the model
expects it to be replaced by the target's backbone-dim embedding,
so the base class's embedding-dim equality guard must not apply.
"""
if get_pp_group().world_size != 1:
return
inner_model = getattr(target_language_model, "model", None)
target_embed_tokens = getattr(inner_model, "embed_tokens", None)
if target_embed_tokens is None:
raise AttributeError(
"Target model does not have an 'embed_tokens' attribute"
)
del self.model.model.embed_tokens
self.model.model.embed_tokens = target_embed_tokens
logger.info(
"Gemma4 MTP: sharing target model's backbone-dim embed_tokens "
"with the draft model."
)
def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None:
"""Gemma4 MTP always keeps its own draft-dim lm_head.
@@ -446,10 +446,16 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
)
last_hidden_states = last_hidden_states[:num_reqs]
sample_positions = positions
if not self.advance_draft_positions:
# The forward pass holds positions fixed (Q-only, shared target KV),
# but Gumbel sampling still needs the absolute draft position.
sample_positions = positions + self.current_draft_step
# Sample the draft tokens.
draft_tokens = self.sample_draft(
last_hidden_states,
positions,
sample_positions,
idx_mapping,
self.temperature,
self.seeds,
@@ -630,6 +636,9 @@ def _prepare_decode_inputs_kernel(
draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride)
tl.store(input_ids_ptr + req_idx, draft_token)
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
num_rejected = tl.load(num_rejected_ptr + req_idx)
seq_len = target_seq_len - num_rejected
if ADVANCE_DRAFT_POSITIONS:
# Compute position and seq_lens.
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
@@ -637,12 +646,8 @@ def _prepare_decode_inputs_kernel(
position = tl.load(positions_ptr + req_idx)
position = tl.minimum(position + 1, max_model_len - 1)
tl.store(positions_ptr + req_idx, position)
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
num_rejected = tl.load(num_rejected_ptr + req_idx)
seq_len = target_seq_len - num_rejected
seq_len = tl.minimum(seq_len + 1, max_model_len)
tl.store(seq_lens_ptr + req_idx, seq_len)
tl.store(seq_lens_ptr + req_idx, seq_len)
def prepare_decode_inputs(