[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-05-06 22:39:29 +08:00
committed by GitHub
co-authored by Luciano Martins
parent f39bcf1e30
commit 27e0057aed
9 changed files with 1120 additions and 71 deletions
+335
View File
@@ -0,0 +1,335 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Gemma4 MTP (Multi-Token Prediction) proposer for speculative decoding.
The Gemma4 assistant model runs all decoder layers per draft step
(producing one token), and all its attention layers share KV cache
with the target model via cross-model KV sharing.
"""
from collections import defaultdict
from copy import copy
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_layers_from_vllm_config, replace
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
class Gemma4Proposer(SpecDecodeBaseProposer):
def __init__(
self,
vllm_config: VllmConfig,
device: torch.device,
runner=None,
):
super().__init__(
vllm_config,
device,
pass_hidden_states_to_model=True,
runner=runner,
)
# All draft steps predict from the same position (the last
# target-model position), so positions and seq_lens must not
# advance between steps.
self.constant_draft_positions = True
# Per-group block tables for multi-group KV cache models.
# Populated by gpu_model_runner during _prepare_inputs.
self._per_group_block_tables: dict[int, torch.Tensor] = {}
# Centroids CUDA graphs — populated in load_model if centroids
# masking is active. _centroids_sizes is pre-sorted for fast
# lookup in _greedy_sample.
self._centroids_sizes: list[int] = []
self._centroids_graphs: dict[int, torch.cuda.CUDAGraph] = {}
self._centroids_inputs: dict[int, torch.Tensor] = {}
self._centroids_outputs: dict[int, torch.Tensor] = {}
def set_per_group_block_table(self, gid: int, block_table: torch.Tensor) -> None:
self._per_group_block_tables[gid] = block_table
def model_returns_tuple(self) -> bool:
# forward() returns (draft_hidden_states, backbone_hidden_states).
# The proposer uses draft_hidden_states for compute_logits and
# backbone_hidden_states for the hidden-state feedback buffer.
return True
def build_per_group_and_layer_attn_metadata(
self,
common_attn_metadata: CommonAttentionMetadata,
draft_index: int = 0,
) -> tuple[list[object], dict[str, object]]:
"""Build attention metadata using the correct block table per group.
Gemma4 has multiple KV cache groups (sliding vs full attention)
with different block tables. The base class receives a single
common_attn_metadata whose block_table belongs to one group.
We swap in the correct block table for each draft attention group.
"""
per_group_attn_metadata: list[object] = []
per_layer_attn_metadata: dict[str, object] = {}
for attn_group in self.draft_attn_groups:
gid = attn_group.kv_cache_group_id
if gid in self._per_group_block_tables:
cm = copy(common_attn_metadata)
cm.block_table_tensor = self._per_group_block_tables[gid]
else:
cm = common_attn_metadata
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
common_attn_metadata=cm, draft_index=draft_index
)
per_group_attn_metadata.append(attn_metadata)
for layer_name in attn_group.layer_names:
per_layer_attn_metadata[layer_name] = attn_metadata
return per_group_attn_metadata, per_layer_attn_metadata
def _greedy_sample(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self._centroids_sizes:
T = hidden_states.shape[0]
for size in self._centroids_sizes:
if size >= T:
self._centroids_inputs[size][:T].copy_(hidden_states)
self._centroids_graphs[size].replay()
return self._centroids_outputs[size][:T].clone()
return self.model.get_top_tokens(hidden_states)
return super()._greedy_sample(hidden_states)
def _setup_centroids_cuda_graphs(self) -> None:
"""Capture CUDA graphs for centroids get_top_tokens at key sizes."""
masked_emb = self.model.masked_embedding
lm_head_weight = self.model._get_full_lm_head_weight()
for size in [1, 2, 4, 8, 16, 32, 64]:
static_input = torch.zeros(
size,
masked_emb.hidden_size,
dtype=self.dtype,
device=self.device,
)
for _ in range(3):
masked_emb.get_top_tokens(static_input, lm_head_weight)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
static_output = masked_emb.get_top_tokens(
static_input,
lm_head_weight,
)
self._centroids_graphs[size] = g
self._centroids_inputs[size] = static_input
self._centroids_outputs[size] = static_output
self._centroids_sizes = sorted(self._centroids_graphs)
logger.info(
"Gemma4 MTP: captured centroids CUDA graphs for sizes %s.",
self._centroids_sizes,
)
def _create_draft_vllm_config(self) -> VllmConfig:
"""Preserve the target's forced TRITON_ATTN backend for draft layers.
Gemma4 forces TRITON_ATTN due to heterogeneous head dimensions
(head_dim=256 sliding, global_head_dim=512 full). The base class
resets attention_config.backend to None for draft models, causing
sliding layers to fall back to FLASH_ATTN which cannot handle
KV-shared cache. Override to carry the target's backend through.
"""
base = super()._create_draft_vllm_config()
target_backend = self.vllm_config.attention_config.backend
if target_backend is not None:
base = replace(
base,
attention_config=replace(
base.attention_config,
backend=target_backend,
),
)
return base
def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None:
"""Gemma4 MTP always keeps its own draft-dim lm_head.
The draft model's lm_head operates in draft hidden_size (e.g. 256),
which differs from the target's backbone hidden_size (e.g. 1536).
Sharing would break compute_logits (and centroids masking when
use_ordered_embeddings is enabled).
"""
logger.info(
"Gemma4 MTP: keeping draft model's own lm_head (draft_dim != backbone_dim)."
)
def load_model(self, target_model: nn.Module) -> None:
target_attn_layer_names = set(
get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
).keys()
)
super().load_model(target_model)
self._setup_gemma4_kv_sharing(target_attn_layer_names)
if getattr(self.model, "masked_embedding", None) is not None:
self._setup_centroids_cuda_graphs()
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
"""Draft layers span multiple KV cache groups (sliding + full
attention with different head dimensions), so skip the base
class single-group assertion."""
def initialize_attn_backend(
self,
kv_cache_config: KVCacheConfig,
kernel_block_sizes: list[int] | None = None,
) -> None:
"""Create separate AttentionGroup objects per KV cache spec
so that each head-dim variant gets its own metadata builder."""
all_attn_layers = get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
)
layer_to_gid: dict[str, int] = {}
layer_to_spec: dict[str, KVCacheSpec] = {}
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
group_spec = group.kv_cache_spec
for ln in group.layer_names:
layer_to_gid[ln] = gid
if isinstance(group_spec, UniformTypeKVCacheSpecs):
if ln in group_spec.kv_cache_specs:
layer_to_spec[ln] = group_spec.kv_cache_specs[ln]
else:
tgt = getattr(
all_attn_layers.get(ln),
"kv_sharing_target_layer_name",
None,
)
if tgt and tgt in group_spec.kv_cache_specs:
layer_to_spec[ln] = group_spec.kv_cache_specs[tgt]
else:
layer_to_spec[ln] = group_spec
else:
layer_to_spec[ln] = group_spec
attention_groups: dict[tuple[tuple[str, str], KVCacheSpec], AttentionGroup] = {}
for layer_name in self._draft_attn_layer_names:
if layer_name not in layer_to_spec:
continue
attn_layer = all_attn_layers[layer_name]
attn_backend = attn_layer.get_attn_backend()
spec = layer_to_spec[layer_name]
gid = layer_to_gid[layer_name]
group_key = (attn_backend.full_cls_name(), spec)
if group_key not in attention_groups:
kernel_block_size = (
kernel_block_sizes[gid]
if kernel_block_sizes is not None and gid < len(kernel_block_sizes)
else None
)
attn_group = AttentionGroup(
backend=attn_backend,
layer_names=[layer_name],
kv_cache_spec=spec,
kv_cache_group_id=gid,
)
attn_group.create_metadata_builders(
self.vllm_config,
self.device,
kernel_block_size=kernel_block_size,
)
attention_groups[group_key] = attn_group
else:
attention_groups[group_key].layer_names.append(layer_name)
self.draft_attn_groups = list(attention_groups.values())
if self.draft_attn_groups:
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
self.block_size = (
self.draft_attn_groups[0]
.get_metadata_builder()
.kv_cache_spec.block_size
)
else:
self.kv_cache_gid = 0
self.block_size = kv_cache_config.kv_cache_groups[
0
].kv_cache_spec.block_size
logger.debug("Using block size %d for drafting layers", self.block_size)
def _setup_gemma4_kv_sharing(
self,
target_attn_layer_names: set[str],
) -> None:
"""Wire draft layers to share KV with the target model.
Each draft decoder layer is mapped to the last non-KV-shared
target layer of the same attention type (sliding or full).
"""
draft_config = self.speculative_config.draft_model_config.hf_config
draft_text_config = draft_config.get_text_config()
target_config = self.vllm_config.model_config.hf_config
target_text_config = target_config.get_text_config()
target_layer_types = getattr(target_text_config, "layer_types", [])
if not (hasattr(self.model, "model") and hasattr(self.model.model, "layers")):
return
target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0)
num_non_shared = len(target_layer_types) - target_num_kv_shared
type_to_target_indices: dict[str, list[int]] = defaultdict(list)
for idx, lt in enumerate(target_layer_types[:num_non_shared]):
type_to_target_indices[lt].append(idx)
target_prefix = "model.layers"
for name in target_attn_layer_names:
if ".layers." in name:
target_prefix = name.split(".layers.")[0] + ".layers"
break
draft_layer_types = getattr(draft_text_config, "layer_types", [])
for draft_idx, layer in enumerate(self.model.model.layers):
if not hasattr(layer, "self_attn"):
continue
attn = getattr(layer.self_attn, "attn", None)
if attn is None:
continue
draft_layer_type = (
draft_layer_types[draft_idx]
if draft_idx < len(draft_layer_types)
else "full_attention"
)
candidates = type_to_target_indices.get(draft_layer_type, [])
if not candidates:
logger.warning(
"No target layer of type '%s' for draft layer %d",
draft_layer_type,
draft_idx,
)
continue
target_idx = candidates[-1]
target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn"
attn.kv_sharing_target_layer_name = target_layer_name
logger.info(
"Gemma4 MTP: draft layer %d (%s) -> %s",
draft_idx,
draft_layer_type,
target_layer_name,
)
+83 -53
View File
@@ -105,6 +105,12 @@ class SpecDecodeBaseProposer:
)
self.needs_extra_input_slots = self.net_num_new_slots_per_request > 0
# When True, all draft steps reuse the same position as the
# first step instead of advancing by one each iteration.
# Used by draft models with Q-only attention that share KV
# with the target and always predict from the same position.
self.constant_draft_positions: bool = False
self.parallel_drafting_token_id: int = 0
self.parallel_drafting_hidden_state_tensor: torch.Tensor | None = None
if self.parallel_drafting:
@@ -389,9 +395,9 @@ class SpecDecodeBaseProposer:
return {name: view for name in self._draft_attn_layer_names}
def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode) -> None:
"""Initialize cudagraph dispatcher keys for eagle.
"""Initialize cudagraph dispatcher keys for the drafter.
Eagle only supports PIECEWISE cudagraphs (via mixed_mode).
Only supports PIECEWISE cudagraphs (via mixed_mode).
This should be called after adjust_cudagraph_sizes_for_spec_decode.
"""
if (
@@ -500,6 +506,12 @@ class SpecDecodeBaseProposer:
positions = self.positions[token_indices_to_sample]
hidden_states = hidden_states[token_indices_to_sample]
if self.constant_draft_positions:
# Write the sampling positions into the front of the
# positions buffer so that subsequent loop iterations
# (which read via _get_positions) use the correct values.
self.positions[:batch_size] = positions
if any(isinstance(md, TreeAttentionMetadata) for md in per_group_attn_metadata):
# Draft using tree attention - requires full logits for top-k
logits = self.model.compute_logits(sample_hidden_states)
@@ -557,59 +569,25 @@ class SpecDecodeBaseProposer:
# cast to int32 is crucial when eagle model is compiled.
# tensor.argmax() returns int64 by default.
input_ids = draft_token_ids_list[-1].int()
# Use fused kernel for slot mapping and metadata updates.
# Write clamped positions directly into the positions buffer to
# avoid an extra D2D copy for the common (non-mrope) case.
positions_1d = positions[0] if self.uses_mrope else positions
if self.uses_mrope:
out_pos = self.mrope_positions[0, :batch_size]
elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:
out_pos = self.xdrope_positions[0, :batch_size]
else:
out_pos = self.positions[:batch_size]
eagle_step_update_slot_mapping_and_metadata(
positions_1d=positions_1d,
block_table_tensor=common_attn_metadata.block_table_tensor,
seq_lens=common_attn_metadata.seq_lens,
block_size=block_size,
max_model_len=self.max_model_len,
out_clamped_positions=out_pos,
out_slot_mapping=self._slot_mapping_buffer[:input_batch_size],
input_batch_size=input_batch_size,
)
common_attn_metadata.slot_mapping = self._slot_mapping_buffer[:batch_size]
if self.uses_mrope:
self.mrope_positions[1:, :batch_size] = self.mrope_positions[
0, :batch_size
]
positions = self.mrope_positions[:, :batch_size]
elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:
self.xdrope_positions[1:, :batch_size] = self.xdrope_positions[
0, :batch_size
]
positions = self.xdrope_positions[0, :batch_size]
else:
positions = self.positions[:batch_size]
# Increment the maximum sequence length. We increment max_seq_len
# unconditionally even though some seq_lens may have been capped above,
# as max_seq_len serves as an upper bound for sequence lengths.
common_attn_metadata.max_seq_len = min(
common_attn_metadata.max_seq_len + 1, self.max_model_len
)
# Also update the CPU-side shadow; NOTE: this is hacky and should be
# removed in when common_attn_metadata.seq_lens_cpu is deprecated.
if common_attn_metadata._seq_lens_cpu is not None:
common_attn_metadata._seq_lens_cpu += 1
if common_attn_metadata._num_computed_tokens_cpu is not None:
common_attn_metadata._num_computed_tokens_cpu += 1
if common_attn_metadata.seq_lens_cpu_upper_bound is not None:
common_attn_metadata.seq_lens_cpu_upper_bound += 1
if not self.constant_draft_positions:
positions = self._update_positions_dependent_metadata(
positions,
common_attn_metadata,
batch_size,
input_batch_size,
block_size,
)
# Rebuild attention metadata
_, per_layer_attn_metadata = self.build_per_group_and_layer_attn_metadata(
common_attn_metadata, draft_index=token_index + 1
)
# Rebuild attention metadata. When draft positions are constant
# (e.g. Gemma4 MTP), common_attn_metadata is invariant across
# loop iterations so we build once and reuse.
if not self.constant_draft_positions or token_index == 0:
_, per_layer_attn_metadata = (
self.build_per_group_and_layer_attn_metadata(
common_attn_metadata, draft_index=token_index + 1
)
)
# copy inputs to buffer for cudagraph
self.input_ids[:batch_size] = input_ids
@@ -655,6 +633,58 @@ class SpecDecodeBaseProposer:
draft_token_ids = torch.stack(draft_token_ids_list, dim=1)
return draft_token_ids
def _update_positions_dependent_metadata(
self,
positions: torch.Tensor,
common_attn_metadata,
batch_size: int,
input_batch_size: int,
block_size: int,
) -> torch.Tensor:
"""Update positions, slot mappings, and sequence metadata for the
next draft step. Returns the updated positions tensor."""
positions_1d = positions[0] if self.uses_mrope else positions
if self.uses_mrope:
out_pos = self.mrope_positions[0, :batch_size]
elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:
out_pos = self.xdrope_positions[0, :batch_size]
else:
out_pos = self.positions[:batch_size]
eagle_step_update_slot_mapping_and_metadata(
positions_1d=positions_1d,
block_table_tensor=common_attn_metadata.block_table_tensor,
seq_lens=common_attn_metadata.seq_lens,
block_size=block_size,
max_model_len=self.max_model_len,
out_clamped_positions=out_pos,
out_slot_mapping=self._slot_mapping_buffer[:input_batch_size],
input_batch_size=input_batch_size,
)
common_attn_metadata.slot_mapping = self._slot_mapping_buffer[:batch_size]
if self.uses_mrope:
self.mrope_positions[1:, :batch_size] = self.mrope_positions[0, :batch_size]
positions = self.mrope_positions[:, :batch_size]
elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0:
self.xdrope_positions[1:, :batch_size] = self.xdrope_positions[
0, :batch_size
]
positions = self.xdrope_positions[0, :batch_size]
else:
positions = self.positions[:batch_size]
common_attn_metadata.max_seq_len = min(
common_attn_metadata.max_seq_len + 1,
self.max_model_len,
)
if common_attn_metadata._seq_lens_cpu is not None:
common_attn_metadata._seq_lens_cpu += 1
if common_attn_metadata._num_computed_tokens_cpu is not None:
common_attn_metadata._num_computed_tokens_cpu += 1
if common_attn_metadata.seq_lens_cpu_upper_bound is not None:
common_attn_metadata.seq_lens_cpu_upper_bound += 1
return positions
def set_inputs_first_pass(
self,
target_token_ids: torch.Tensor,