[Tests][Spec Decode] Add gemma4 MTP acceptance rates test (#47920)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -188,3 +188,15 @@ steps:
|
||||
- tests/v1/e2e/spec_decode/test_mtp_parallel_load.py
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/test_mtp_parallel_load.py
|
||||
|
||||
- label: Spec Decode Acceptance Rates Nightly
|
||||
key: spec-decode-acceptance-rates-nightly
|
||||
timeout_in_minutes: 60
|
||||
device: h200_35gb
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "acceptance_rates"
|
||||
|
||||
@@ -1393,51 +1393,81 @@ def load_and_process_dataset(data_name: str):
|
||||
return dataset
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dflash_config():
|
||||
target_model = "Qwen/Qwen3-8B"
|
||||
draft_model = "z-lab/Qwen3-8B-DFlash-b16"
|
||||
|
||||
return dict(
|
||||
model=target_model,
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "dflash",
|
||||
"model": draft_model,
|
||||
"num_speculative_tokens": 16,
|
||||
"max_model_len": 32768,
|
||||
},
|
||||
max_model_len=32768,
|
||||
max_num_seqs=128,
|
||||
gpu_memory_utilization=0.85,
|
||||
enforce_eager=False,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["spec_config", "expected_acceptance_lengths", "chat_template_kwargs"],
|
||||
[
|
||||
pytest.param(
|
||||
dict(
|
||||
model="Qwen/Qwen3-8B",
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "dflash",
|
||||
"model": "z-lab/Qwen3-8B-DFlash-b16",
|
||||
"num_speculative_tokens": 16,
|
||||
"max_model_len": 32768,
|
||||
},
|
||||
max_model_len=32768,
|
||||
max_num_seqs=128,
|
||||
gpu_memory_utilization=0.85,
|
||||
enforce_eager=False,
|
||||
disable_log_stats=False,
|
||||
),
|
||||
# All scores from Table 1 in https://arxiv.org/pdf/2602.06036
|
||||
{
|
||||
"mt-bench": 4.24,
|
||||
"humaneval": 6.50,
|
||||
# runs with a subset of prompts so extra wide tol here
|
||||
"gsm8k": 6.54 * 0.975,
|
||||
},
|
||||
{"enable_thinking": False},
|
||||
id="dflash",
|
||||
),
|
||||
pytest.param(
|
||||
dict(
|
||||
model="google/gemma-4-E4B-it",
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"model": "google/gemma-4-E4B-it-assistant",
|
||||
"num_speculative_tokens": 2,
|
||||
"max_model_len": 32768,
|
||||
},
|
||||
max_model_len=32768,
|
||||
# Skip multimodal profiling; this is a text-only eval.
|
||||
limit_mm_per_prompt={"image": 0, "audio": 0},
|
||||
disable_log_stats=False,
|
||||
),
|
||||
{
|
||||
"mt-bench": 2.28,
|
||||
"humaneval": 2.68,
|
||||
# runs with a subset of prompts so extra wide tol here
|
||||
"gsm8k": 2.67 * 0.975,
|
||||
},
|
||||
{},
|
||||
id="gemma4",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_mrv2", [False, True])
|
||||
def test_dflash_acceptance_rates(
|
||||
monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config
|
||||
def test_acceptance_rates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
spec_config: dict[str, Any],
|
||||
expected_acceptance_lengths: dict[str, float],
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
use_mrv2: bool,
|
||||
):
|
||||
"""
|
||||
E2E test for DFlash (block diffusion) speculative decoding.
|
||||
Runs acceptance rate validation on GSM8k, MT-Bench, and HumanEval
|
||||
comparing against baseline results from the paper (Table 1).
|
||||
See https://github.com/z-lab/dflash/blob/main/benchmark_sglang.py for methodology.
|
||||
E2E acceptance-rate validation for speculative decoding.
|
||||
|
||||
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
|
||||
through the spec decode engine and asserts the mean acceptance length
|
||||
stays within tolerance of the reference figure for each dataset.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
|
||||
|
||||
spec_llm = LLM(**dflash_config)
|
||||
spec_llm = LLM(**spec_config)
|
||||
|
||||
max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k
|
||||
|
||||
# All scores from Table 1 in https://arxiv.org/pdf/2602.06036
|
||||
expected_acceptance_lengths = {
|
||||
"mt-bench": 4.24,
|
||||
"humaneval": 6.50,
|
||||
"gsm8k": 6.54 * 0.975, # runs with a subset of prompts so extra wide tol here
|
||||
}
|
||||
|
||||
tokenizer = spec_llm.get_tokenizer()
|
||||
for dataset_name, expected_len in expected_acceptance_lengths.items():
|
||||
dataset = load_and_process_dataset(dataset_name)
|
||||
@@ -1452,10 +1482,11 @@ def test_dflash_acceptance_rates(
|
||||
[{"role": "user", "content": user_content}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# Temp=0, MaxTokens=2048 from the paper
|
||||
# Greedy (temp=0) so acceptance length is deterministic and comparable
|
||||
# across runs.
|
||||
spec_llm.generate(
|
||||
[prompt_text],
|
||||
SamplingParams(temperature=0, max_tokens=2048),
|
||||
@@ -1467,17 +1498,17 @@ def test_dflash_acceptance_rates(
|
||||
acceptance_lengths.append(acceptance_len)
|
||||
|
||||
mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths)
|
||||
# Fairly tight tolerance of 95% against the paper's figures,
|
||||
# Fairly tight tolerance of 95% against the reference figures,
|
||||
# watching for regressions. Can be relaxed if test is flaky but be sure to
|
||||
# check for genuine issues such as #40727.
|
||||
expected_len = expected_len * 0.95
|
||||
print(
|
||||
f"DFlash acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
|
||||
f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
|
||||
f" (expected at least {expected_len:.2f})"
|
||||
)
|
||||
|
||||
assert mean_acceptance_length >= expected_len, (
|
||||
f"DFlash acceptance_len for {dataset_name} is below expected threshold:"
|
||||
f"acceptance_len for {dataset_name} is below expected threshold: "
|
||||
f"{mean_acceptance_length:.2f} < {expected_len:.2f}"
|
||||
)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
@@ -165,6 +166,29 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
)
|
||||
return base
|
||||
|
||||
def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None:
|
||||
"""Gemma4 MTP requires dim-mismatched embedding sharing.
|
||||
|
||||
The draft checkpoint's embed_tokens is a draft-dim placeholder
|
||||
(tied to lm_head so load_weights populates both); the model
|
||||
expects it to be replaced by the target's backbone-dim embedding,
|
||||
so the base class's embedding-dim equality guard must not apply.
|
||||
"""
|
||||
if get_pp_group().world_size != 1:
|
||||
return
|
||||
inner_model = getattr(target_language_model, "model", None)
|
||||
target_embed_tokens = getattr(inner_model, "embed_tokens", None)
|
||||
if target_embed_tokens is None:
|
||||
raise AttributeError(
|
||||
"Target model does not have an 'embed_tokens' attribute"
|
||||
)
|
||||
del self.model.model.embed_tokens
|
||||
self.model.model.embed_tokens = target_embed_tokens
|
||||
logger.info(
|
||||
"Gemma4 MTP: sharing target model's backbone-dim embed_tokens "
|
||||
"with the draft model."
|
||||
)
|
||||
|
||||
def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None:
|
||||
"""Gemma4 MTP always keeps its own draft-dim lm_head.
|
||||
|
||||
|
||||
@@ -446,10 +446,16 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
last_hidden_states = last_hidden_states[:num_reqs]
|
||||
|
||||
sample_positions = positions
|
||||
if not self.advance_draft_positions:
|
||||
# The forward pass holds positions fixed (Q-only, shared target KV),
|
||||
# but Gumbel sampling still needs the absolute draft position.
|
||||
sample_positions = positions + self.current_draft_step
|
||||
|
||||
# Sample the draft tokens.
|
||||
draft_tokens = self.sample_draft(
|
||||
last_hidden_states,
|
||||
positions,
|
||||
sample_positions,
|
||||
idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
@@ -630,6 +636,9 @@ def _prepare_decode_inputs_kernel(
|
||||
draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride)
|
||||
tl.store(input_ids_ptr + req_idx, draft_token)
|
||||
|
||||
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
seq_len = target_seq_len - num_rejected
|
||||
if ADVANCE_DRAFT_POSITIONS:
|
||||
# Compute position and seq_lens.
|
||||
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
|
||||
@@ -637,12 +646,8 @@ def _prepare_decode_inputs_kernel(
|
||||
position = tl.load(positions_ptr + req_idx)
|
||||
position = tl.minimum(position + 1, max_model_len - 1)
|
||||
tl.store(positions_ptr + req_idx, position)
|
||||
|
||||
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
seq_len = target_seq_len - num_rejected
|
||||
seq_len = tl.minimum(seq_len + 1, max_model_len)
|
||||
tl.store(seq_lens_ptr + req_idx, seq_len)
|
||||
tl.store(seq_lens_ptr + req_idx, seq_len)
|
||||
|
||||
|
||||
def prepare_decode_inputs(
|
||||
|
||||
Reference in New Issue
Block a user