diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py index 45f2a13fdd..9c47826850 100644 --- a/tests/evals/gsm8k/gsm8k_eval.py +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -283,28 +283,36 @@ def evaluate_gsm8k_offline( max_tokens: int = 256, temperature: float = 0.0, gen_prefix: str = "", + use_chat_completions: bool = False, ) -> dict[str, float | int]: """Evaluate GSM8K accuracy using an offline vllm.LLM object. Same prompts and scoring as evaluate_gsm8k(), but runs generation directly via llm.generate() instead of calling a server over HTTP. + + When ``use_chat_completions=True``, prompts go through the chat template via + ``llm.chat()`` instead of raw completion (for instruction-tuned models). """ from vllm import SamplingParams prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) - sampling_params = SamplingParams( temperature=temperature, max_tokens=max_tokens, stop=["Question", "Assistant:", "<|separator|>"], ) - + mode = "chat" if use_chat_completions else "completion" print( - f"Running offline GSM8K evaluation: {len(prompts)} questions, {num_shots}-shot" + f"Running offline GSM8K evaluation: {len(prompts)} questions, " + f"{num_shots}-shot, {mode}" ) tic = time.perf_counter() - outputs = llm.generate(prompts, sampling_params) + if use_chat_completions: + conversations = [[{"role": "user", "content": p}] for p in prompts] + outputs = llm.chat(conversations, sampling_params) + else: + outputs = llm.generate(prompts, sampling_params) latency = time.perf_counter() - tic states = [o.outputs[0].text for o in outputs] diff --git a/tests/models/registry.py b/tests/models/registry.py index a05362f42b..fa23e14d96 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1477,6 +1477,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { is_available_online=False, use_original_num_layers=True, # DSpark backbone requires all layers ), + "Gemma4DSparkModel": _HfExamplesInfo( + "google/gemma-4-12B-it", + speculative_model="deepseek-ai/dspark_gemma4_12b_block7", + is_available_online=False, + use_original_num_layers=True, + ), # [Eagle] "EagleCohereForCausalLM": _HfExamplesInfo( "/host/engines/cohere-moe", diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index 96d6684594..01bec69640 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -286,6 +286,73 @@ def test_suffix_decoding_acceptance( cleanup_dist_env_and_memory() +@pytest.mark.slow_test +@large_gpu_mark(min_gb=80) +def test_gemma4_dspark_correctness_and_acceptance_rate( + monkeypatch: pytest.MonkeyPatch, +): + """ + E2E test for Gemma4 DSpark speculative decoding: acceptance rate/length + regression coverage plus GSM8K correctness, at temperature=1.0 to exercise + the probabilistic draft-sampling/rejection-sampling path (not just greedy). + + Uses google/gemma-4-12B-it as target with the dspark_gemma4_12b_block7 draft + model. Exercises the full Gemma4 DSpark path (draft build over the reused + base classes DFlashQwen3Model / Qwen3DSparkForCausalLM / Gemma4MTP* / + DSparkMarkovHead, the fused context-KV precompute, the Markov head, and + rejection sampling), so it fails if any base class drifts out of sync. + + gemma-4-12B is instruct-tuned, so GSM8K is run through the chat template + (use_chat_completions=True; raw few-shot completion collapses to a few + percent). Reference: measured over 5 runs of 200 GSM8K questions at + temperature=1.0 (prefix caching disabled): + accuracy: min=0.900 max=0.955 mean=0.937 + acceptance_rate: min=0.578 max=0.595 mean=0.588 + acceptance_len: min=5.044 max=5.167 mean=5.116 + Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling + """ + monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0") + + spec_llm = LLM( + model="google/gemma-4-12B-it", + trust_remote_code=True, + speculative_config={ + "method": "dspark", + "model": "deepseek-ai/dspark_gemma4_12b_block7", + "num_speculative_tokens": 7, + "draft_sample_method": "probabilistic", + }, + max_model_len=8192, + max_num_seqs=32, + gpu_memory_utilization=0.8, + enforce_eager=True, + enable_prefix_caching=False, + disable_log_stats=False, + ) + try: + results = evaluate_gsm8k_offline( + spec_llm, num_questions=200, temperature=1.0, use_chat_completions=True + ) + gsm8k_accuracy = results["accuracy"] + + metrics = spec_llm.get_metrics() + acceptance_rate = compute_acceptance_rate(metrics) + acceptance_len = compute_acceptance_len(metrics) + print( + f"Gemma4 DSpark acceptance_rate={acceptance_rate:.2f}, " + f"acceptance_len={acceptance_len:.2f}, " + f"gsm8k_accuracy={gsm8k_accuracy:.3f}" + ) + + assert acceptance_rate >= 0.588 * 0.9 + assert acceptance_len >= 5.116 * 0.9 + assert gsm8k_accuracy >= 0.937 * 0.9 + finally: + del spec_llm + torch.accelerator.empty_cache() + cleanup_dist_env_and_memory() + + @pytest.mark.parametrize( ["model_path", "expected_accuracy_threshold"], [ diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index cf6f0a5ae6..8ba55a2ec9 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -882,6 +882,7 @@ class SpeculativeConfig: elif ( "dspark" in self.draft_model_config.model.lower() or "Qwen3DSparkModel" in self.draft_model_config.architectures + or "Gemma4DSparkModel" in self.draft_model_config.architectures ): self.method = "dspark" elif self.draft_model_config.hf_config.model_type == "medusa": @@ -932,6 +933,7 @@ class SpeculativeConfig: if self.method == "dspark" and ( "Qwen3DSparkModel" not in self.draft_model_config.architectures + and "Gemma4DSparkModel" not in self.draft_model_config.architectures ): # DeepSeek-V4 DSpark reuses the full DeepSeek-V4 config # and its weights ship in the target checkpoint. @@ -940,6 +942,23 @@ class SpeculativeConfig: "DSparkDraftModel" ] self.update_arch_() + elif ( + self.method == "dspark" + and "Gemma4DSparkModel" in self.draft_model_config.architectures + ): + # Normalize the self-contained Gemma4 draft's config keys to + # the DSpark conventions. + hf = self.draft_model_config.hf_config + if ( + getattr(hf, "dspark_target_layer_ids", None) is None + and getattr(hf, "target_layer_ids", None) is not None + ): + hf.dspark_target_layer_ids = hf.target_layer_ids + if ( + getattr(hf, "n_predict", None) is None + and getattr(hf, "block_size", None) is not None + ): + hf.n_predict = hf.block_size if self.method in ("dflash", "dspark"): self.parallel_drafting = True diff --git a/vllm/model_executor/models/gemma4_dspark.py b/vllm/model_executor/models/gemma4_dspark.py new file mode 100644 index 0000000000..792f108842 --- /dev/null +++ b/vllm/model_executor/models/gemma4_dspark.py @@ -0,0 +1,312 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Gemma4 DSpark draft model for speculative decoding.""" + +from collections.abc import Iterable + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from vllm import _custom_ops as ops +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer +from .qwen3_dflash import DFlashQwen3Model +from .qwen3_dspark import DSparkMarkovHead, Qwen3DSparkForCausalLM +from .utils import extract_layer_index, maybe_prefix + + +class Gemma4DSparkAttention(Gemma4MTPAttention): + """Gemma4 attention with its own KV cache and K/V projections.""" + + def __init__( + self, + config, + cache_config: CacheConfig | None, + quant_config: QuantizationConfig | None, + prefix: str, + ) -> None: + is_full = config.layer_types[extract_layer_index(prefix)] == "full_attention" + head_dim = ( + getattr(config, "global_head_dim", config.head_dim) + if is_full + else config.head_dim + ) + use_k_eq_v = is_full and getattr(config, "attention_k_eq_v", False) + num_kv_heads = ( + getattr(config, "num_global_key_value_heads", config.num_key_value_heads) + if use_k_eq_v + else config.num_key_value_heads + ) + super().__init__( + config=config, + hidden_size=config.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + max_position_embeddings=config.max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + attn_logits_soft_cap=getattr(config, "attn_logit_softcapping", None), + prefix=prefix, + ) + self.is_kv_shared_layer = False + self.use_k_eq_v = use_k_eq_v + self.kv_size = self.num_kv_heads * self.head_dim + attn_bias = getattr(config, "attention_bias", False) + self.k_proj = ColumnParallelLinear( + config.hidden_size, + self.total_num_kv_heads * self.head_dim, + bias=attn_bias, + quant_config=quant_config, + prefix=f"{prefix}.k_proj", + ) + self.v_proj = ( + None + if use_k_eq_v + else ColumnParallelLinear( + config.hidden_size, + self.total_num_kv_heads * self.head_dim, + bias=attn_bias, + quant_config=quant_config, + prefix=f"{prefix}.v_proj", + ) + ) + self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.v_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, has_weight=False) + + def _kv_proj( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + k, _ = self.k_proj(hidden_states) + k_normed = self.k_norm(k.unflatten(-1, (self.num_kv_heads, self.head_dim))) + v_src = k if self.use_k_eq_v else self.v_proj(hidden_states)[0] + v_normed = self.v_norm(v_src.unflatten(-1, (self.num_kv_heads, self.head_dim))) + return k_normed.flatten(-2, -1), v_normed.flatten(-2, -1) + + def forward( + self, positions: torch.Tensor, hidden_states: torch.Tensor, **kwargs + ) -> torch.Tensor: + q, _ = self.q_proj(hidden_states) + q = self.q_norm(q.unflatten(-1, (self.num_heads, self.head_dim))).flatten( + -2, -1 + ) + k, v = self._kv_proj(hidden_states) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class Gemma4DSparkDecoderLayer(Gemma4MTPDecoderLayer): + """Gemma4 MTP decoder layer using the KV-owning DSpark attention.""" + + def __init__( + self, + config, + cache_config: CacheConfig | None, + quant_config: QuantizationConfig | None, + prefix: str, + ) -> None: + super().__init__( + config, cache_config=cache_config, quant_config=quant_config, prefix=prefix + ) + get_current_vllm_config().compilation_config.static_forward_context.pop( + f"{prefix}.self_attn.attn", None + ) + self.self_attn = Gemma4DSparkAttention( + config, cache_config, quant_config, prefix=f"{prefix}.self_attn" + ) + + +@support_torch_compile +class Gemma4DSparkModel(DFlashQwen3Model): + """Gemma4 DSpark draft backbone (Gemma4 layers + DSpark Markov head).""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + nn.Module.__init__(self) + assert vllm_config.speculative_config is not None + config = vllm_config.speculative_config.draft_model_config.hf_config + self.config = config + self.use_aux_hidden_state = True + self.target_layer_ids = tuple( + getattr(config, "dspark_target_layer_ids", None) or config.target_layer_ids + ) + current_vllm_config = get_current_vllm_config() + cache_config = current_vllm_config.cache_config + quant_config = current_vllm_config.quant_config + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + prefix=maybe_prefix(prefix, "embed_tokens"), + ) + self.register_buffer( + "normalizer", + torch.tensor(config.hidden_size**0.5, dtype=vllm_config.model_config.dtype), + persistent=False, + ) + self.fc = ReplicatedLinear( + config.hidden_size * len(self.target_layer_ids), + config.hidden_size, + bias=False, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "fc"), + ) + self.hidden_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.layers = nn.ModuleList( + Gemma4DSparkDecoderLayer( + config, + cache_config, + quant_config, + prefix=maybe_prefix(prefix, f"layers.{i}"), + ) + for i in range(config.num_hidden_layers) + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + draft_vocab_size = ( + getattr(config, "draft_vocab_size", None) or config.vocab_size + ) + self.markov_head = DSparkMarkovHead( + config.vocab_size, + draft_vocab_size, + config.markov_rank, + prefix=maybe_prefix(prefix, "markov_head"), + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) * self.normalizer + + def _build_fused_kv_buffers(self) -> None: + layers_attn = [layer.self_attn for layer in self.layers] + attn0 = layers_attn[0] + assert all(a.use_k_eq_v for a in layers_attn), ( + "Gemma4 DSpark fused precompute assumes uniform attention_k_eq_v layers" + ) + self._build_context_kv_buffers(layers_attn, attn0.k_proj.bias is not None) + self._rope_head_size = attn0.rotary_emb.head_size + self._rope_cos_sin_cache = attn0.rotary_emb.cos_sin_cache + self._rope_is_neox = attn0.rotary_emb.is_neox_style + self._num_attn_layers = len(layers_attn) + self._kv_size = attn0.kv_size + self._head_dim = attn0.head_dim + self._num_kv_heads = attn0.num_kv_heads + self._rms_norm_eps = attn0.q_norm.variance_epsilon + self._attn_layers = [layer.self_attn.attn for layer in self.layers] + + def _build_context_kv_buffers( + self, layers_attn: list[nn.Module], has_bias: bool + ) -> None: + self._hidden_norm_weight = self.hidden_norm.weight.data + self._fused_k_weight = torch.cat([a.k_proj.weight for a in layers_attn], dim=0) + self._fused_k_bias: torch.Tensor | None = ( + torch.cat([a.k_proj.bias for a in layers_attn], dim=0) if has_bias else None + ) + self._k_norm_weights = torch.stack( + [a.k_norm.weight.data for a in layers_attn], dim=0 + ).contiguous() + # v_norm has no learnable scale; ones matching the K-norm call shape. + self._v_norm_weights = torch.ones( + len(layers_attn), + layers_attn[0].head_dim, + dtype=self._k_norm_weights.dtype, + device=self._k_norm_weights.device, + ) + + def _project_context_kv( + self, + context_states: torch.Tensor, + num_ctx: int, + num_layers: int, + num_kv_heads: int, + head_dim: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + # Project once via k_proj for all layers. K is raw (the inherited path + # applies k_norm + RoPE); V = v_norm(same projection), no RoPE (k_eq_v). + normed = torch.empty_like(context_states) + ops.rms_norm( + normed, context_states, self._hidden_norm_weight, self._rms_norm_eps + ) + all_k_flat = F.linear(normed, self._fused_k_weight, self._fused_k_bias) + all_k = ( + all_k_flat.view(num_ctx, num_layers, num_kv_heads, head_dim) + .permute(1, 0, 2, 3) + .contiguous() + ) + all_v = torch.empty_like(all_k) + ops.rms_norm(all_v, all_k, self._v_norm_weights, self._rms_norm_eps) + return all_k, all_v + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + input_embeds: torch.Tensor | None = None, + ) -> torch.Tensor: + hidden_states = ( + self.embed_input_ids(input_ids) if input_embeds is None else input_embeds + ) + for layer in self.layers: + hidden_states, _ = layer(positions, hidden_states, None) + return self.norm(hidden_states) + + +class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM): + """Gemma4 DSpark speculator over a self-contained draft checkpoint.""" + + dspark_shares_target_embeddings = False + packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]} + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + nn.Module.__init__(self) + assert vllm_config.speculative_config is not None + self.draft_model_config = vllm_config.speculative_config.draft_model_config + self.config = self.draft_model_config.hf_config + self.model = Gemma4DSparkModel( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + self.lm_head = ParallelLMHead( + self.config.vocab_size, + self.config.hidden_size, + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = LogitsProcessor( + self.config.vocab_size, + soft_cap=getattr(self.config, "final_logit_softcapping", None), + ) + self.draft_id_to_target_id = None + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked = [("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "up_proj", 1)] + params = dict(self.named_parameters()) + params.update(dict(self.named_buffers())) + loaded: set[str] = set() + for name, w in weights: + if "confidence_head" in name: + continue + if "lm_head" not in name: + name = "model." + name + for pn, wn, shard in stacked: + if wn in name and (mapped := name.replace(wn, pn)) in params: + params[mapped].weight_loader(params[mapped], w, shard) + loaded.add(mapped) + break + else: + if name in params: + p = params[name] + getattr(p, "weight_loader", default_weight_loader)(p, w) + loaded.add(name) + self.model._build_fused_kv_buffers() + return loaded diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index b5ec2519d0..d28aac974b 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -609,6 +609,7 @@ _SPECULATIVE_DECODING_MODELS = { "DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"), "Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"), "DFlashLagunaForCausalLM": ("laguna_dflash", "DFlashLagunaForCausalLM"), + "Gemma4DSparkModel": ("gemma4_dspark", "Gemma4DSparkForCausalLM"), "PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "PeagleLlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),