From 272abd5f486967f1fb9db7ca7504f8c34235ef50 Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:15:54 -0700 Subject: [PATCH] [Tests][Spec Decode] Add gemma4 MTP acceptance rates test (#47920) Signed-off-by: Giancarlo Delfin --- .buildkite/test_areas/spec_decode.yaml | 12 ++ tests/v1/e2e/spec_decode/test_spec_decode.py | 115 +++++++++++------- vllm/v1/spec_decode/gemma4.py | 24 ++++ .../spec_decode/autoregressive/speculator.py | 17 ++- 4 files changed, 120 insertions(+), 48 deletions(-) diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 977562ee97..c63aaa18d8 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -188,3 +188,15 @@ steps: - tests/v1/e2e/spec_decode/test_mtp_parallel_load.py commands: - pytest -v -s v1/e2e/spec_decode/test_mtp_parallel_load.py + +- label: Spec Decode Acceptance Rates Nightly + key: spec-decode-acceptance-rates-nightly + timeout_in_minutes: 60 + device: h200_35gb + optional: true + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "acceptance_rates" diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index 01bec69640..17c48721f8 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -1393,51 +1393,81 @@ def load_and_process_dataset(data_name: str): return dataset -@pytest.fixture -def dflash_config(): - target_model = "Qwen/Qwen3-8B" - draft_model = "z-lab/Qwen3-8B-DFlash-b16" - - return dict( - model=target_model, - trust_remote_code=True, - speculative_config={ - "method": "dflash", - "model": draft_model, - "num_speculative_tokens": 16, - "max_model_len": 32768, - }, - max_model_len=32768, - max_num_seqs=128, - gpu_memory_utilization=0.85, - enforce_eager=False, - disable_log_stats=False, - ) - - +@pytest.mark.parametrize( + ["spec_config", "expected_acceptance_lengths", "chat_template_kwargs"], + [ + pytest.param( + dict( + model="Qwen/Qwen3-8B", + trust_remote_code=True, + speculative_config={ + "method": "dflash", + "model": "z-lab/Qwen3-8B-DFlash-b16", + "num_speculative_tokens": 16, + "max_model_len": 32768, + }, + max_model_len=32768, + max_num_seqs=128, + gpu_memory_utilization=0.85, + enforce_eager=False, + disable_log_stats=False, + ), + # All scores from Table 1 in https://arxiv.org/pdf/2602.06036 + { + "mt-bench": 4.24, + "humaneval": 6.50, + # runs with a subset of prompts so extra wide tol here + "gsm8k": 6.54 * 0.975, + }, + {"enable_thinking": False}, + id="dflash", + ), + pytest.param( + dict( + model="google/gemma-4-E4B-it", + trust_remote_code=True, + speculative_config={ + "method": "mtp", + "model": "google/gemma-4-E4B-it-assistant", + "num_speculative_tokens": 2, + "max_model_len": 32768, + }, + max_model_len=32768, + # Skip multimodal profiling; this is a text-only eval. + limit_mm_per_prompt={"image": 0, "audio": 0}, + disable_log_stats=False, + ), + { + "mt-bench": 2.28, + "humaneval": 2.68, + # runs with a subset of prompts so extra wide tol here + "gsm8k": 2.67 * 0.975, + }, + {}, + id="gemma4", + ), + ], +) @pytest.mark.parametrize("use_mrv2", [False, True]) -def test_dflash_acceptance_rates( - monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config +def test_acceptance_rates( + monkeypatch: pytest.MonkeyPatch, + spec_config: dict[str, Any], + expected_acceptance_lengths: dict[str, float], + chat_template_kwargs: dict[str, Any], + use_mrv2: bool, ): """ - E2E test for DFlash (block diffusion) speculative decoding. - Runs acceptance rate validation on GSM8k, MT-Bench, and HumanEval - comparing against baseline results from the paper (Table 1). - See https://github.com/z-lab/dflash/blob/main/benchmark_sglang.py for methodology. + E2E acceptance-rate validation for speculative decoding. + + Drives one or more datasets (keyed in ``expected_acceptance_lengths``) + through the spec decode engine and asserts the mean acceptance length + stays within tolerance of the reference figure for each dataset. """ monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0") - - spec_llm = LLM(**dflash_config) + spec_llm = LLM(**spec_config) max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k - # All scores from Table 1 in https://arxiv.org/pdf/2602.06036 - expected_acceptance_lengths = { - "mt-bench": 4.24, - "humaneval": 6.50, - "gsm8k": 6.54 * 0.975, # runs with a subset of prompts so extra wide tol here - } - tokenizer = spec_llm.get_tokenizer() for dataset_name, expected_len in expected_acceptance_lengths.items(): dataset = load_and_process_dataset(dataset_name) @@ -1452,10 +1482,11 @@ def test_dflash_acceptance_rates( [{"role": "user", "content": user_content}], tokenize=False, add_generation_prompt=True, - enable_thinking=False, + **chat_template_kwargs, ) - # Temp=0, MaxTokens=2048 from the paper + # Greedy (temp=0) so acceptance length is deterministic and comparable + # across runs. spec_llm.generate( [prompt_text], SamplingParams(temperature=0, max_tokens=2048), @@ -1467,17 +1498,17 @@ def test_dflash_acceptance_rates( acceptance_lengths.append(acceptance_len) mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths) - # Fairly tight tolerance of 95% against the paper's figures, + # Fairly tight tolerance of 95% against the reference figures, # watching for regressions. Can be relaxed if test is flaky but be sure to # check for genuine issues such as #40727. expected_len = expected_len * 0.95 print( - f"DFlash acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}" + f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}" f" (expected at least {expected_len:.2f})" ) assert mean_acceptance_length >= expected_len, ( - f"DFlash acceptance_len for {dataset_name} is below expected threshold:" + f"acceptance_len for {dataset_name} is below expected threshold: " f"{mean_acceptance_length:.2f} < {expected_len:.2f}" ) diff --git a/vllm/v1/spec_decode/gemma4.py b/vllm/v1/spec_decode/gemma4.py index 7f67ae9f49..07cfaa2f4d 100644 --- a/vllm/v1/spec_decode/gemma4.py +++ b/vllm/v1/spec_decode/gemma4.py @@ -14,6 +14,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig, get_layers_from_vllm_config, replace +from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.v1.attention.backend import CommonAttentionMetadata @@ -165,6 +166,29 @@ class Gemma4Proposer(SpecDecodeBaseProposer): ) return base + def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None: + """Gemma4 MTP requires dim-mismatched embedding sharing. + + The draft checkpoint's embed_tokens is a draft-dim placeholder + (tied to lm_head so load_weights populates both); the model + expects it to be replaced by the target's backbone-dim embedding, + so the base class's embedding-dim equality guard must not apply. + """ + if get_pp_group().world_size != 1: + return + inner_model = getattr(target_language_model, "model", None) + target_embed_tokens = getattr(inner_model, "embed_tokens", None) + if target_embed_tokens is None: + raise AttributeError( + "Target model does not have an 'embed_tokens' attribute" + ) + del self.model.model.embed_tokens + self.model.model.embed_tokens = target_embed_tokens + logger.info( + "Gemma4 MTP: sharing target model's backbone-dim embed_tokens " + "with the draft model." + ) + def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None: """Gemma4 MTP always keeps its own draft-dim lm_head. diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index b890574971..9035ea38b2 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -446,10 +446,16 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): ) last_hidden_states = last_hidden_states[:num_reqs] + sample_positions = positions + if not self.advance_draft_positions: + # The forward pass holds positions fixed (Q-only, shared target KV), + # but Gumbel sampling still needs the absolute draft position. + sample_positions = positions + self.current_draft_step + # Sample the draft tokens. draft_tokens = self.sample_draft( last_hidden_states, - positions, + sample_positions, idx_mapping, self.temperature, self.seeds, @@ -630,6 +636,9 @@ def _prepare_decode_inputs_kernel( draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride) tl.store(input_ids_ptr + req_idx, draft_token) + target_seq_len = tl.load(target_seq_lens_ptr + req_idx) + num_rejected = tl.load(num_rejected_ptr + req_idx) + seq_len = target_seq_len - num_rejected if ADVANCE_DRAFT_POSITIONS: # Compute position and seq_lens. # NOTE(woosuk): To prevent out-of-range access, we clamp these values @@ -637,12 +646,8 @@ def _prepare_decode_inputs_kernel( position = tl.load(positions_ptr + req_idx) position = tl.minimum(position + 1, max_model_len - 1) tl.store(positions_ptr + req_idx, position) - - target_seq_len = tl.load(target_seq_lens_ptr + req_idx) - num_rejected = tl.load(num_rejected_ptr + req_idx) - seq_len = target_seq_len - num_rejected seq_len = tl.minimum(seq_len + 1, max_model_len) - tl.store(seq_lens_ptr + req_idx, seq_len) + tl.store(seq_lens_ptr + req_idx, seq_len) def prepare_decode_inputs(