[Model Runner V2][Spec Decode] Add Gemma4 MTP support (#43241)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-06-04 00:51:06 +00:00
committed by GitHub
parent 0414d75410
commit ceb0111a90
14 changed files with 1243 additions and 942 deletions
+3 -5
View File
@@ -62,7 +62,6 @@ from vllm.v1.attention.backends.utils import (
KVCacheLayoutType,
get_dcp_local_seq_lens,
get_kv_cache_layout,
get_num_attention_heads_from_layers,
get_per_layer_parameters,
infer_global_hyperparameters,
split_decodes_and_prefills,
@@ -608,10 +607,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
# Compatible with models with non-uniform per-layer head counts.
self.num_qo_heads = get_num_attention_heads_from_layers(
vllm_config, layer_names
) or self.model_config.get_num_attention_heads(self.vllm_config.parallel_config)
self.num_qo_heads = self.model_config.get_num_attention_heads(
self.vllm_config.parallel_config
)
self.num_kv_heads = self.kv_cache_spec.num_kv_heads
self.head_dim = self.kv_cache_spec.head_size
+4 -8
View File
@@ -30,10 +30,7 @@ from vllm.v1.attention.backend import (
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.utils import (
get_kv_cache_layout,
get_num_attention_heads_from_layers,
)
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.attention.ops.triton_prefill_attention import context_attention_fwd
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash,
@@ -142,10 +139,9 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
self.block_size = kv_cache_spec.block_size
model_config = vllm_config.model_config
# Compatible with models with non-uniform per-layer head counts.
self.num_heads_q = get_num_attention_heads_from_layers(
vllm_config, layer_names
) or model_config.get_num_attention_heads(vllm_config.parallel_config)
self.num_heads_q = model_config.get_num_attention_heads(
vllm_config.parallel_config
)
self.num_heads_kv = model_config.get_num_kv_heads(vllm_config.parallel_config)
self.headdim = model_config.get_head_size()
-26
View File
@@ -136,32 +136,6 @@ def get_per_layer_parameters(
return per_layer_params
def get_num_attention_heads_from_layers(
vllm_config: VllmConfig, layer_names: list[str]
) -> int | None:
"""Per-TP-rank ``num_heads`` shared by the named Attention layers.
Use in metadata builders whose plan-time allocations depend on the
head count: the model-wide ``get_num_attention_heads()`` is wrong
for models with non-uniform per-layer head counts. All layers in
one attention group must agree on ``num_heads``; this is asserted.
Returns ``None`` when no matching Attention layer is found.
"""
attn_layers = get_layers_from_vllm_config(
vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
layer_names,
)
if not attn_layers:
return None
heads = {layer.impl.num_heads for layer in attn_layers.values()}
assert len(heads) == 1, (
f"All layers in one attention group must share num_heads; "
f"got {heads} for {layer_names}."
)
return heads.pop()
def infer_global_hyperparameters(
per_layer_params: dict[str, PerLayerParameters],
) -> PerLayerParameters:
+3 -2
View File
@@ -103,6 +103,7 @@ from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
set_eagle3_aux_hidden_state_layers,
)
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
@@ -307,7 +308,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.use_aux_hidden_state_outputs:
assert self.speculative_config is not None
set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config)
if self.speculator is not None:
if isinstance(self.speculator, DraftModelSpeculator):
self.speculator.load_model(self.model)
eplb_models_added = self.eplb.maybe_register_speculator(
self.speculator, self.speculative_config, load_dummy_weights
@@ -457,7 +458,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.init_cudagraph_manager(cudagraph_mode)
check_attention_cp_compatibility(self.vllm_config)
if self.speculator is not None:
if isinstance(self.speculator, DraftModelSpeculator):
# HACK(woosuk)
self.speculator.set_attn(
self.model_state, self.kv_cache_config, self.block_tables
+16 -3
View File
@@ -8,8 +8,21 @@ from vllm.config import VllmConfig
def init_speculator(vllm_config: VllmConfig, device: torch.device):
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
if speculative_config.use_eagle():
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator
if speculative_config.use_gemma4_mtp():
from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import (
Gemma4Speculator,
)
return Gemma4Speculator(vllm_config, device)
elif speculative_config.method == "mtp":
from vllm.v1.worker.gpu.spec_decode.mtp.speculator import MTPSpeculator
return MTPSpeculator(vllm_config, device)
elif speculative_config.use_eagle():
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import (
EagleSpeculator,
)
return EagleSpeculator(vllm_config, device)
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
else:
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -19,8 +19,8 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup
class PrefillEagleCudaGraphManager(CudaGraphManager):
"""Eagle CudaGraphManager for prefill, using pre-built attention states
class PrefillSpeculatorCudaGraphManager(CudaGraphManager):
"""CudaGraphManager for draft prefill, using pre-built attention states
from the target model's capture."""
def capture(
@@ -56,9 +56,8 @@ class PrefillEagleCudaGraphManager(CudaGraphManager):
super().capture(create_forward_fn, progress_bar_desc)
class DecodeEagleCudaGraphManager(CudaGraphManager):
"""Eagle CudaGraphManager for decode draft generation, building its own
attention metadata from scratch."""
class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
"""CudaGraphManager for draft decode, building its own attention metadata."""
def capture(
self,
@@ -0,0 +1,795 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionStatePair,
BatchExecutionDescriptor,
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
DecodeSpeculatorCudaGraphManager,
PrefillSpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
logger = init_logger(__name__)
class AutoRegressiveSpeculator(DraftModelSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
self.hidden_states = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device)
self.last_token_indices = torch.zeros(
self.max_num_reqs, dtype=torch.int64, device=device
)
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
self.draft_model_config
)
if self.supports_mm_inputs:
self.inputs_embeds = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
self.prefill_cudagraph_manager: PrefillSpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: DecodeSpeculatorCudaGraphManager | None = None
@property
def advance_draft_positions(self) -> bool:
"""
Whether to increment positions and seq_lens between draft steps.
True for Eagle/standard MTP (each step produces new KV).
False for Gemma4 MTP (Q-only, shares target KV, constant positions).
"""
return True
@property
def model_returns_tuple(self) -> bool:
"""
Whether the draft model's forward() returns a tuple.
True: returns (last_hidden_states, hidden_states) — Eagle, Gemma4 MTP.
False: returns a single tensor used for both — standard MTP (DeepSeek).
"""
return True
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# Initialize cudagraph manager for draft prefill (draft position 0).
self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
self.num_speculative_steps + 1,
)
# PIECEWISE cudagraphs are not supported for draft decodes.
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
else:
cudagraph_mode = CUDAGraphMode.NONE
# Initialize cudagraph manager for draft decodes (draft positions > 0).
self.decode_cudagraph_manager = DecodeSpeculatorCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
decode_query_len=1,
)
def capture(
self,
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
) -> None:
logger.info("Capturing model for speculator...")
# Reset indices to zeros to prevent stale values from prior
# dummy runs to cause out-of-bounds indexing during capture.
self.last_token_indices.zero_()
# Capture the prefill routine (model forward + compute_logits +
# sample).
# For FULL graphs, the entire routine is recorded as one graph.
# For PIECEWISE, only the model's compiled regions are captured
# and the rest (compute_logits, gumbel_sample) runs eagerly.
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.prefill_cudagraph_manager.capture(
self._prefill,
attn_states,
progress_bar_desc="Capturing prefill CUDA graphs",
)
if self.num_speculative_steps == 1:
return
# Capture the decode draft generation routine (model forward +
# sample + update_draft_inputs) for a single
# step.
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.capture(
self._generate_draft,
self.model_state,
self.input_buffers,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
progress_bar_desc="Capturing decode CUDA graphs",
)
@torch.inference_mode()
def propose(
self,
input_batch: InputBatch,
attn_metadata: dict[str, Any],
slot_mappings: dict[str, torch.Tensor],
# [num_tokens, hidden_size]
last_hidden_states: torch.Tensor,
# num_layers x [num_tokens, hidden_size]
aux_hidden_states: list[torch.Tensor] | None,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_tokens = input_batch.num_tokens_after_padding
num_reqs = input_batch.num_reqs
max_query_len = input_batch.num_scheduled_tokens.max()
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
self.draft_max_seq_len = min(
max_seq_len + self.num_speculative_steps, self.max_model_len
)
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
# number of rejected tokens, we maintain the size of input_ids and
# hidden_states the same as the target model's. This means, we pad each
# request's query length to include any rejected positions. By doing so,
# we can also reuse the attention metadata (e.g., query_start_loc,
# seq_lens) of the target model.
if aux_hidden_states:
assert self.method == "eagle3"
hidden_states = self.model.combine_hidden_states(
torch.cat(aux_hidden_states, dim=-1)
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_tokens].copy_(hidden_states)
self._copy_request_inputs(
num_reqs,
input_batch.idx_mapping,
temperature,
seeds,
)
# Get the input ids and last token indices for the speculator.
prepare_prefill_inputs(
self.last_token_indices,
self.current_draft_step,
self.input_buffers,
input_batch,
num_sampled,
num_rejected,
last_sampled,
next_prefill_tokens,
self.max_num_reqs,
)
# When all requests are decoding (no true prefills), each has
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
uniform_token_count = get_uniform_token_count(
num_reqs,
# Use the actual number of tokens without padding added by
# the target model during FULL cudagraph.
input_batch.num_tokens,
max_query_len,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
num_reqs,
num_tokens,
uniform_token_count,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
# Replay the full graph for draft prefill.
assert self.prefill_cudagraph_manager is not None
self.prefill_cudagraph_manager.run_fullgraph(prefill_batch_desc)
else:
# The target model's attention metadata and slot mappings
# can directly be used for draft prefill, because of the
# identical batch shape and KV cache layout.
self._prefill(
num_reqs,
prefill_batch_desc.num_tokens,
attn_metadata,
slot_mappings,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=prefill_batch_desc.cg_mode,
mm_inputs=mm_inputs,
)
if self.num_speculative_steps == 1:
# Early exit.
return self.draft_tokens[:num_reqs, :1]
# Prepare the inputs for the decode steps.
prepare_decode_inputs(
self.draft_tokens[:num_reqs, 0],
input_batch.seq_lens,
num_rejected,
self.input_buffers,
self.max_model_len,
self.max_num_reqs,
advance_draft_positions=self.advance_draft_positions,
)
# Each request produces exactly 1 token per draft generation step,
# enabling FULL graph replay.
decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.decode_cudagraph_manager,
num_reqs,
num_reqs,
uniform_token_count=1,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
# Generate the remaining num_speculative_steps - 1 draft tokens.
self._multi_step_decode(
num_reqs,
dummy_run and skip_attn_for_dummy_run,
decode_batch_desc,
num_tokens_across_dp,
)
return self.draft_tokens[:num_reqs]
def sample_draft(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
) -> torch.Tensor:
logits = self.model.compute_logits(hidden_states)
if draft_logits is not None:
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
# used for draft and target sampling.
return gumbel_sample(
logits,
idx_mapping,
temperature,
seeds,
positions + 1,
apply_temperature=True,
output_processed_logits=draft_logits,
output_processed_logits_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
else:
return logits.argmax(dim=-1)
@torch.inference_mode()
def _run_model(
self,
num_tokens: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
with set_forward_context(
attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
cudagraph_runtime_mode=cudagraph_runtime_mode,
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
):
inputs_embeds = None
if self.supports_mm_inputs:
# Merge multimodal embeddings with input ids.
mm_embeds, is_mm_embed = mm_inputs or (None, None)
num_input_tokens = (
is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens
)
self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids(
self.input_buffers.input_ids[:num_input_tokens],
multimodal_embeddings=mm_embeds,
is_multimodal=is_mm_embed,
)
inputs_embeds = self.inputs_embeds[:num_tokens]
model_inputs = dict(
input_ids=self.input_buffers.input_ids[:num_tokens],
positions=self.input_buffers.positions[:num_tokens],
hidden_states=self.hidden_states[:num_tokens],
inputs_embeds=inputs_embeds,
)
if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
# Draft prefill with PIECEWISE cudagraph (compiled PW or breakable),
# chosen inside run_pw_graph.
assert self.prefill_cudagraph_manager is not None
ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph(
self.model, model_inputs
)
else:
# Eager (NONE): call the raw model directly.
ret_hidden_states = self.model(**model_inputs)
if self.model_returns_tuple:
last_hidden_states, hidden_states = ret_hidden_states
else:
last_hidden_states = ret_hidden_states
hidden_states = ret_hidden_states
return last_hidden_states, hidden_states
def _prefill(
self,
num_reqs: int,
num_tokens: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
) -> None:
last_token_indices = self.last_token_indices[:num_reqs]
positions = self.input_buffers.positions[last_token_indices]
idx_mapping = self.idx_mapping[:num_reqs]
last_hidden_states, hidden_states = self._run_model(
num_tokens,
attn_metadata,
slot_mappings,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=cudagraph_runtime_mode,
mm_inputs=mm_inputs,
)
sample_hidden_states = last_hidden_states[last_token_indices]
self.draft_tokens[:num_reqs, 0] = self.sample_draft(
sample_hidden_states,
positions,
idx_mapping,
self.temperature,
self.seeds,
self.current_draft_step,
self.draft_logits,
)
self.hidden_states[:num_reqs] = hidden_states[last_token_indices]
self.input_buffers.positions[:num_reqs] = positions
def _multi_step_decode(
self,
num_reqs: int,
skip_attn: bool,
batch_desc: BatchExecutionDescriptor,
num_tokens_across_dp: torch.Tensor | None,
) -> None:
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
idx_mapping = self.idx_mapping[:num_reqs]
attn_metadata = None
slot_mappings_by_layer = None
for step in range(1, self.num_speculative_steps):
# Rebuild every step when positions advance, or just once
# on the first step when positions are constant (Gemma4 MTP).
if not skip_attn and (self.advance_draft_positions or step == 1):
slot_mappings = self.block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
batch_desc.num_tokens,
)
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
attn_metadata = self._build_draft_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=batch_desc.num_reqs or num_reqs,
num_tokens_padded=batch_desc.num_tokens,
)
# Update the current draft step.
self.current_draft_step.fill_(step)
# Generate draft tokens for the current step.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
else:
self._generate_draft(
num_reqs,
batch_desc.num_tokens,
attn_metadata,
slot_mappings_by_layer,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=batch_desc.cg_mode,
)
def _generate_draft(
self,
num_reqs: int,
num_tokens_padded: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> None:
idx_mapping = self.idx_mapping[:num_reqs]
positions = self.input_buffers.positions[:num_reqs]
# Run the draft model forward pass.
last_hidden_states, hidden_states = self._run_model(
num_tokens_padded,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
)
last_hidden_states = last_hidden_states[:num_reqs]
# Sample the draft tokens.
draft_tokens = self.sample_draft(
last_hidden_states,
positions,
idx_mapping,
self.temperature,
self.seeds,
self.current_draft_step,
self.draft_logits,
)
# Update the inputs for the next step.
update_draft_inputs(
draft_tokens,
self.current_draft_step,
hidden_states,
self.draft_tokens,
self.hidden_states,
self.input_buffers,
num_reqs,
self.max_model_len,
self.num_speculative_steps,
advance_draft_positions=self.advance_draft_positions,
)
@triton.jit
def _prepare_prefill_inputs_kernel(
last_token_indices_ptr,
draft_current_step_ptr,
draft_input_ids_ptr,
draft_positions_ptr,
draft_query_start_loc_ptr,
draft_seq_lens_ptr,
target_input_ids_ptr,
target_positions_ptr,
idx_mapping_ptr,
last_sampled_ptr,
next_prefill_tokens_ptr,
num_sampled_ptr,
num_rejected_ptr,
query_start_loc_ptr,
seq_lens_ptr,
max_num_reqs,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
num_reqs = tl.num_programs(0)
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
query_start = tl.load(query_start_loc_ptr + req_idx)
query_end = tl.load(query_start_loc_ptr + req_idx + 1)
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + req_idx)
# Get the true query length and next token after accounting for rejected tokens.
num_rejected = tl.load(num_rejected_ptr + req_idx)
query_len -= num_rejected
num_sampled = tl.load(num_sampled_ptr + req_idx)
if num_sampled > 0:
next_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32)
else:
# Chunked prefilling.
# Get the next prefill token.
next_token = tl.load(next_prefill_tokens_ptr + req_state_idx)
# Shift target_input_ids by one.
for i in range(1, query_len, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < query_len
input_ids = tl.load(target_input_ids_ptr + query_start + block, mask=mask)
tl.store(draft_input_ids_ptr + query_start + block - 1, input_ids, mask=mask)
last_token_index = query_start + query_len - 1
tl.store(last_token_indices_ptr + req_idx, last_token_index)
tl.store(draft_input_ids_ptr + last_token_index, next_token)
# Copy positions.
for i in range(0, query_len, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < query_len
target_pos = tl.load(target_positions_ptr + query_start + block, mask=mask)
tl.store(draft_positions_ptr + query_start + block, target_pos, mask=mask)
# Copy query start locations.
tl.store(draft_query_start_loc_ptr + req_idx, query_start)
# Copy sequence lengths.
tl.store(draft_seq_lens_ptr + req_idx, seq_len)
if req_idx == (num_reqs - 1):
# Reset the current draft step to 0.
tl.store(draft_current_step_ptr, 0)
# Pad query_start_loc for CUDA graphs.
for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs + 1
tl.store(draft_query_start_loc_ptr + block, query_end, mask=mask)
# Pad seq_lens for CUDA graphs.
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(draft_seq_lens_ptr + block, 0, mask=mask)
# Pad last_token_indices for CUDA graphs.
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(last_token_indices_ptr + block, 0, mask=mask)
def prepare_prefill_inputs(
# [num_reqs]
last_token_indices: torch.Tensor,
current_draft_step: torch.Tensor,
input_buffers: InputBuffers,
input_batch: InputBatch,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
max_num_reqs,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
_prepare_prefill_inputs_kernel[(num_reqs,)](
last_token_indices,
current_draft_step,
input_buffers.input_ids,
input_buffers.positions,
input_buffers.query_start_loc,
input_buffers.seq_lens,
input_batch.input_ids,
input_batch.positions,
input_batch.idx_mapping,
last_sampled,
next_prefill_tokens,
num_sampled,
num_rejected,
input_batch.query_start_loc,
input_batch.seq_lens,
max_num_reqs,
BLOCK_SIZE=1024,
)
return last_token_indices
@triton.jit
def _prepare_decode_inputs_kernel(
draft_tokens_ptr,
draft_tokens_stride,
target_seq_lens_ptr,
num_rejected_ptr,
input_ids_ptr,
positions_ptr,
query_start_loc_ptr,
seq_lens_ptr,
max_model_len,
max_num_reqs,
BLOCK_SIZE: tl.constexpr,
ADVANCE_DRAFT_POSITIONS: tl.constexpr,
):
req_idx = tl.program_id(0)
num_reqs = tl.num_programs(0) - 1
if req_idx == num_reqs:
# Compute query_start_loc. Pad it with the last query_start_loc
# for CUDA graphs.
for i in range(0, max_num_reqs + 1, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
q = tl.where(block < num_reqs, block, num_reqs)
mask = block < max_num_reqs + 1
tl.store(query_start_loc_ptr + block, q, mask=mask)
# Pad seq_lens for CUDA graphs.
for i in range(req_idx, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(seq_lens_ptr + block, 0, mask=mask)
return
# draft token -> input id.
draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride)
tl.store(input_ids_ptr + req_idx, draft_token)
if ADVANCE_DRAFT_POSITIONS:
# Compute position and seq_lens.
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
# if they reach the max model length.
position = tl.load(positions_ptr + req_idx)
position = tl.minimum(position + 1, max_model_len - 1)
tl.store(positions_ptr + req_idx, position)
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
num_rejected = tl.load(num_rejected_ptr + req_idx)
seq_len = target_seq_len - num_rejected
seq_len = tl.minimum(seq_len + 1, max_model_len)
tl.store(seq_lens_ptr + req_idx, seq_len)
def prepare_decode_inputs(
draft_tokens: torch.Tensor,
target_seq_lens: torch.Tensor,
num_rejected: torch.Tensor,
input_buffers: InputBuffers,
max_model_len: int,
max_num_reqs: int,
advance_draft_positions: bool = True,
):
num_reqs = draft_tokens.shape[0]
_prepare_decode_inputs_kernel[(num_reqs + 1,)](
draft_tokens,
draft_tokens.stride(0),
target_seq_lens,
num_rejected,
input_buffers.input_ids,
input_buffers.positions,
input_buffers.query_start_loc,
input_buffers.seq_lens,
max_model_len,
max_num_reqs,
BLOCK_SIZE=1024,
ADVANCE_DRAFT_POSITIONS=advance_draft_positions,
)
@triton.jit
def _update_draft_inputs_kernel(
output_draft_tokens_ptr,
output_draft_tokens_stride,
next_input_hidden_states_ptr,
next_input_hidden_states_stride,
input_ids_ptr,
positions_ptr,
seq_lens_ptr,
draft_tokens_ptr,
current_draft_step_ptr,
hidden_states_ptr,
hidden_states_stride,
hidden_size,
max_model_len,
num_speculative_steps,
BLOCK_SIZE: tl.constexpr,
ADVANCE_DRAFT_POSITIONS: tl.constexpr,
):
req_idx = tl.program_id(0)
# Write the sampled draft token into self.draft_tokens[req_idx, step].
draft_token = tl.load(draft_tokens_ptr + req_idx)
step = tl.load(current_draft_step_ptr)
tl.store(
output_draft_tokens_ptr + req_idx * output_draft_tokens_stride + step,
draft_token,
)
if step >= num_speculative_steps - 1:
# This is the final step. Skip updating draft forward inputs.
return
# Write the sampled draft token into the input ids tensor for the next
# forward pass.
tl.store(input_ids_ptr + req_idx, draft_token)
# Copy hidden states into the input hidden states tensor for the next
# forward pass.
for i in range(0, hidden_size, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < hidden_size
hidden_states = tl.load(
hidden_states_ptr + req_idx * hidden_states_stride + block,
mask=mask,
)
tl.store(
next_input_hidden_states_ptr
+ req_idx * next_input_hidden_states_stride
+ block,
hidden_states,
mask=mask,
)
if ADVANCE_DRAFT_POSITIONS:
# Increment position and seq_lens.
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
# if they reach the max model length.
position = tl.load(positions_ptr + req_idx)
position = tl.minimum(position + 1, max_model_len - 1)
tl.store(positions_ptr + req_idx, position)
seq_len = tl.load(seq_lens_ptr + req_idx)
seq_len = tl.minimum(seq_len + 1, max_model_len)
tl.store(seq_lens_ptr + req_idx, seq_len)
def update_draft_inputs(
draft_tokens: torch.Tensor,
current_draft_step: torch.Tensor,
hidden_states: torch.Tensor,
output_draft_tokens: torch.Tensor,
next_input_hidden_states: torch.Tensor,
input_buffers: InputBuffers,
num_reqs: int,
max_model_len: int,
num_speculative_steps: int,
advance_draft_positions: bool = True,
):
_, hidden_size = hidden_states.shape
_update_draft_inputs_kernel[(num_reqs,)](
output_draft_tokens,
output_draft_tokens.stride(0),
next_input_hidden_states,
next_input_hidden_states.stride(0),
input_buffers.input_ids,
input_buffers.positions,
input_buffers.seq_lens,
draft_tokens,
current_draft_step,
hidden_states,
hidden_states.stride(0),
hidden_size,
max_model_len,
num_speculative_steps,
BLOCK_SIZE=1024,
ADVANCE_DRAFT_POSITIONS=advance_draft_positions,
)
@@ -1,903 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.triton_utils import tl, triton
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
build_slot_mappings_by_layer,
init_attn_backend,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionStatePair,
BatchExecutionDescriptor,
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.eagle.cudagraph import (
DecodeEagleCudaGraphManager,
PrefillEagleCudaGraphManager,
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
logger = init_logger(__name__)
class EagleSpeculator:
def __init__(self, vllm_config: VllmConfig, device: torch.device):
self.vllm_config = vllm_config
self.device = device
self.speculative_config = vllm_config.speculative_config
assert self.speculative_config is not None
self.method = self.speculative_config.method
self.num_speculative_steps = self.speculative_config.num_speculative_tokens
self.draft_model_config = self.speculative_config.draft_model_config
self.scheduler_config = vllm_config.scheduler_config
self.max_num_reqs = self.scheduler_config.max_num_seqs
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
self.max_model_len = vllm_config.model_config.max_model_len
self.draft_max_seq_len = self.max_model_len
# We need to get the hidden size from the draft model config because
# the draft model's hidden size can be different from the target model's
# hidden size (e.g., Llama 3.3 70B).
self.hidden_size = self.draft_model_config.get_hidden_size()
# Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP
# draft the target's pre-hc_head (T, hc_mult * hidden_size) residual).
# Non-HC models default to hc_mult=1 and are unaffected.
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
self.hidden_size = self.hidden_size * hc_mult
self.vocab_size = self.draft_model_config.get_vocab_size()
self.dtype = vllm_config.model_config.dtype
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
# DP configuration
self.dp_size = vllm_config.parallel_config.data_parallel_size
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
self.input_buffers = InputBuffers(
max_num_reqs=self.max_num_reqs,
max_num_tokens=self.max_num_tokens,
device=device,
)
self.hidden_states = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
self.idx_mapping = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=device
)
self.temperature = torch.zeros(
self.max_num_reqs, dtype=torch.float32, device=device
)
self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device)
self.draft_tokens = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps,
dtype=torch.int64,
device=device,
)
self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device)
self.last_token_indices = torch.zeros(
self.max_num_reqs, dtype=torch.int64, device=device
)
self.arange = torch.arange(
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
)
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
self.draft_model_config
)
if self.supports_mm_inputs:
self.inputs_embeds = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
self.draft_logits: torch.Tensor | None = None
if self.speculative_config.draft_sample_method == "probabilistic":
self.draft_logits = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps,
self.vocab_size,
dtype=torch.float32,
device=device,
)
self.prefill_cudagraph_manager: PrefillEagleCudaGraphManager | None = None
self.decode_cudagraph_manager: DecodeEagleCudaGraphManager | None = None
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
cudagraph_mode = self.vllm_config.compilation_config.cudagraph_mode
# Initialize cudagraph manager for draft prefill (draft position 0).
self.prefill_cudagraph_manager = PrefillEagleCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
self.num_speculative_steps + 1,
)
# PIECEWISE cudagraphs are not supported for eagle draft decodes.
# PIECEWISE pads num_tokens to the next capture size without padding
# num_reqs, which can cause attention backends to read past the
# valid per-request metadata (e.g. FlashInfer's kv_indptr buffer).
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
else:
cudagraph_mode = CUDAGraphMode.NONE
# Initialize cudagraph manager for draft decodes (draft positions > 0).
self.decode_cudagraph_manager = DecodeEagleCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
decode_query_len=1,
)
def load_model(self, target_model: nn.Module) -> None:
target_attn_layer_names = get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
).keys()
self.model = load_eagle_model(target_model, self.vllm_config)
all_attn_layers = get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
).keys()
self.draft_attn_layer_names = set(all_attn_layers) - set(
target_attn_layer_names
)
def set_attn(
class EagleSpeculator(AutoRegressiveSpeculator):
def load_draft_model(
self,
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
) -> None:
self.model_state = model_state
self.kv_cache_config = kv_cache_config
self.attn_groups, _, _ = init_attn_backend(
kv_cache_config,
self.vllm_config,
self.device,
active_layer_names=self.draft_attn_layer_names,
)
self.block_tables = block_tables
@torch.inference_mode()
def run_model(
self,
num_tokens: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
with set_forward_context(
attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
cudagraph_runtime_mode=cudagraph_runtime_mode,
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
):
inputs_embeds = None
if self.supports_mm_inputs:
# Merge multimodal embeddings with input ids.
mm_embeds, is_mm_embed = mm_inputs or (None, None)
num_input_tokens = (
is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens
)
self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids(
self.input_buffers.input_ids[:num_input_tokens],
multimodal_embeddings=mm_embeds,
is_multimodal=is_mm_embed,
)
inputs_embeds = self.inputs_embeds[:num_tokens]
model_inputs = dict(
input_ids=self.input_buffers.input_ids[:num_tokens],
positions=self.input_buffers.positions[:num_tokens],
hidden_states=self.hidden_states[:num_tokens],
inputs_embeds=inputs_embeds,
)
if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
# Draft prefill with PIECEWISE cudagraph (compiled PW or breakable),
# chosen inside run_pw_graph.
assert self.prefill_cudagraph_manager is not None
ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph(
self.model, model_inputs
)
else:
# Eager (NONE): call the raw model directly.
ret_hidden_states = self.model(**model_inputs)
if self.method == "mtp":
last_hidden_states = ret_hidden_states
hidden_states = ret_hidden_states
else:
last_hidden_states, hidden_states = ret_hidden_states
return last_hidden_states, hidden_states
def _sample_draft(
self,
logits: torch.Tensor,
idx_mapping: torch.Tensor,
pos: torch.Tensor,
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
) -> torch.Tensor:
if draft_logits is not None:
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
# used for draft and target sampling.
return gumbel_sample(
logits,
idx_mapping,
self.temperature,
self.seeds,
pos + 1,
apply_temperature=True,
output_processed_logits=draft_logits,
output_processed_logits_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
else:
return logits.argmax(dim=-1)
def prefill(
self,
num_reqs: int,
num_tokens: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
) -> None:
last_token_indices = self.last_token_indices[:num_reqs]
pos = self.input_buffers.positions[last_token_indices]
idx_mapping = self.idx_mapping[:num_reqs]
last_hidden_states, hidden_states = self.run_model(
num_tokens,
attn_metadata,
slot_mappings,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=cudagraph_runtime_mode,
mm_inputs=mm_inputs,
)
sample_hidden_states = last_hidden_states[last_token_indices]
logits = self.model.compute_logits(sample_hidden_states)
self.draft_tokens[:num_reqs, 0] = self._sample_draft(
logits,
idx_mapping,
pos,
self.current_draft_step,
self.draft_logits,
)
self.hidden_states[:num_reqs] = hidden_states[last_token_indices]
self.input_buffers.positions[:num_reqs] = pos
def multi_step_decode(
self,
num_reqs: int,
skip_attn: bool,
batch_desc: BatchExecutionDescriptor,
num_tokens_across_dp: torch.Tensor | None,
) -> None:
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
idx_mapping = self.idx_mapping[:num_reqs]
for step in range(1, self.num_speculative_steps):
attn_metadata = None
slot_mappings_by_layer = None
if not skip_attn:
# Build attention metadata and slot mappings for each draft
# decode step. It is necessary to rebuild the attention
# metadata even when replaying the FULL graph so that any
# attention metadata builder state is updated.
slot_mappings = self.block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
batch_desc.num_tokens,
)
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
attn_metadata = self._build_draft_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=batch_desc.num_reqs or num_reqs,
num_tokens_padded=batch_desc.num_tokens,
)
# Update the current draft step.
self.current_draft_step.fill_(step)
# Generate draft tokens for the current step.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
else:
self.generate_draft(
num_reqs,
batch_desc.num_tokens,
attn_metadata,
slot_mappings_by_layer,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=batch_desc.cg_mode,
)
def generate_draft(
self,
num_reqs: int,
num_tokens_padded: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> None:
idx_mapping = self.idx_mapping[:num_reqs]
positions = self.input_buffers.positions[:num_reqs]
# Run the eagle model forward pass.
last_hidden_states, hidden_states = self.run_model(
num_tokens_padded,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
)
last_hidden_states = last_hidden_states[:num_reqs]
# Sample the draft tokens.
logits = self.model.compute_logits(last_hidden_states)
draft_tokens = self._sample_draft(
logits,
idx_mapping,
positions,
self.current_draft_step,
self.draft_logits,
)
# Update the inputs for the next step.
update_eagle_draft_inputs(
draft_tokens,
self.current_draft_step,
hidden_states,
self.draft_tokens,
self.hidden_states,
self.input_buffers,
num_reqs,
self.max_model_len,
self.num_speculative_steps,
)
def _build_draft_attn_metadata(
self,
num_reqs: int,
num_reqs_padded: int,
num_tokens_padded: int,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
query_start_loc_cpu = torch.clamp(
self.arange[: num_reqs_padded + 1], max=num_reqs
)
block_tables = [
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
]
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded]
attn_metadata = build_attn_metadata(
attn_groups=self.attn_groups,
num_reqs=num_reqs_padded,
num_tokens=num_tokens_padded,
query_start_loc_gpu=self.input_buffers.query_start_loc[
: num_reqs_padded + 1
],
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=1,
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
max_seq_len=self.draft_max_seq_len,
block_tables=block_tables,
slot_mappings=slot_mappings,
kv_cache_config=self.kv_cache_config,
)
return attn_metadata
def capture(
self,
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
) -> None:
logger.info("Capturing model for Eagle speculator...")
# Reset indices to zeros to prevent stale values from prior
# dummy runs to cause out-of-bounds indexing during capture.
self.last_token_indices.zero_()
# Capture the prefill routine (model forward + compute_logits +
# sample).
# For FULL graphs, the entire routine is recorded as one graph.
# For PIECEWISE, only the model's compiled regions are captured
# and the rest (compute_logits, gumbel_sample) runs eagerly.
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.prefill_cudagraph_manager.capture(
self.prefill,
attn_states,
progress_bar_desc="Capturing eagle prefill CUDA graphs",
)
if self.num_speculative_steps == 1:
return
# Capture the decode draft generation routine (model forward +
# compute_logits + sample + update_eagle_inputs) for a single
# step.
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.capture(
self.generate_draft,
self.model_state,
self.input_buffers,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
progress_bar_desc="Capturing eagle decode CUDA graphs",
)
@torch.inference_mode()
def propose(
self,
input_batch: InputBatch,
attn_metadata: dict[str, Any],
slot_mappings: dict[str, torch.Tensor],
# [num_tokens, hidden_size]
last_hidden_states: torch.Tensor,
# num_layers x [num_tokens, hidden_size]
aux_hidden_states: list[torch.Tensor] | None,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_tokens = input_batch.num_tokens_after_padding
num_reqs = input_batch.num_reqs
max_query_len = input_batch.num_scheduled_tokens.max()
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
self.draft_max_seq_len = min(
max_seq_len + self.num_speculative_steps, self.max_model_len
)
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
# number of rejected tokens, we maintain the size of eagle's input_ids and
# hidden_states the same as the target model's. This means, we pad each
# request's query length to include any rejected positions. By doing so,
# we can also reuse the attention metadata (e.g., query_start_loc,
# seq_lens) of the target model.
if aux_hidden_states:
assert self.method == "eagle3"
hidden_states = self.model.combine_hidden_states(
torch.cat(aux_hidden_states, dim=-1)
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_tokens].copy_(hidden_states)
# Copy temperature, seeds, and idx mapping to the pre-allocated buffers.
# NOTE(woosuk): For draft sampling, we only consider the temperature
# and ignore the other sampling parameters such as top_k and top_p,
# for simplicity and performance.
# While this may slightly degrade the acceptance rate, it does not
# affect the output distribution after rejection sampling.
self.temperature.copy_(temperature)
self.seeds.copy_(seeds)
self.idx_mapping[:num_reqs].copy_(input_batch.idx_mapping)
# Get the input ids and last token indices for the speculator.
prepare_eagle_inputs(
self.last_token_indices,
self.current_draft_step,
self.input_buffers,
input_batch,
num_sampled,
num_rejected,
last_sampled,
next_prefill_tokens,
self.max_num_reqs,
)
# When all requests are decoding (no true prefills), each has
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
uniform_token_count = get_uniform_token_count(
num_reqs,
# Use the actual number of tokens without padding added by
# the target model during FULL cudagraph.
input_batch.num_tokens,
max_query_len,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
num_reqs,
num_tokens,
uniform_token_count,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
# Replay the full graph for draft prefill.
assert self.prefill_cudagraph_manager is not None
self.prefill_cudagraph_manager.run_fullgraph(prefill_batch_desc)
else:
# The target model's attention metadata and slot mappings
# can directly be used for draft prefill, because of the
# identical batch shape and KV cache layout.
self.prefill(
num_reqs,
prefill_batch_desc.num_tokens,
attn_metadata,
slot_mappings,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=prefill_batch_desc.cg_mode,
mm_inputs=mm_inputs,
)
if self.num_speculative_steps == 1:
# Early exit.
return self.draft_tokens[:num_reqs, :1]
# Prepare the inputs for the decode steps.
prepare_eagle_decode(
self.draft_tokens[:num_reqs, 0],
input_batch.seq_lens,
num_rejected,
self.input_buffers,
self.max_model_len,
self.max_num_reqs,
)
# Each request produces exactly 1 token per draft generation step,
# enabling FULL graph replay.
decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.decode_cudagraph_manager,
num_reqs,
num_reqs,
uniform_token_count=1,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
# Generate the remaining num_speculative_steps - 1 draft tokens.
self.multi_step_decode(
num_reqs,
dummy_run and skip_attn_for_dummy_run,
decode_batch_desc,
num_tokens_across_dp,
)
return self.draft_tokens[:num_reqs]
@triton.jit
def _prepare_eagle_inputs_kernel(
last_token_indices_ptr,
eagle_current_draft_step_ptr,
eagle_input_ids_ptr,
eagle_positions_ptr,
eagle_query_start_loc_ptr,
eagle_seq_lens_ptr,
target_input_ids_ptr,
target_positions_ptr,
idx_mapping_ptr,
last_sampled_ptr,
next_prefill_tokens_ptr,
num_sampled_ptr,
num_rejected_ptr,
query_start_loc_ptr,
seq_lens_ptr,
max_num_reqs,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
num_reqs = tl.num_programs(0)
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
query_start = tl.load(query_start_loc_ptr + req_idx)
query_end = tl.load(query_start_loc_ptr + req_idx + 1)
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + req_idx)
# Get the true query length and next token after accounting for rejected tokens.
num_rejected = tl.load(num_rejected_ptr + req_idx)
query_len -= num_rejected
num_sampled = tl.load(num_sampled_ptr + req_idx)
if num_sampled > 0:
next_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32)
else:
# Chunked prefilling.
# Get the next prefill token.
next_token = tl.load(next_prefill_tokens_ptr + req_state_idx)
# Shift target_input_ids by one.
for i in range(1, query_len, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < query_len
input_ids = tl.load(target_input_ids_ptr + query_start + block, mask=mask)
tl.store(eagle_input_ids_ptr + query_start + block - 1, input_ids, mask=mask)
last_token_index = query_start + query_len - 1
tl.store(last_token_indices_ptr + req_idx, last_token_index)
tl.store(eagle_input_ids_ptr + last_token_index, next_token)
# Copy positions.
for i in range(0, query_len, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < query_len
target_pos = tl.load(target_positions_ptr + query_start + block, mask=mask)
tl.store(eagle_positions_ptr + query_start + block, target_pos, mask=mask)
# Copy query start locations.
tl.store(eagle_query_start_loc_ptr + req_idx, query_start)
# Copy sequence lengths.
tl.store(eagle_seq_lens_ptr + req_idx, seq_len)
if req_idx == (num_reqs - 1):
# Reset the current draft step to 0.
tl.store(eagle_current_draft_step_ptr, 0)
# Pad query_start_loc for CUDA graphs.
for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs + 1
tl.store(eagle_query_start_loc_ptr + block, query_end, mask=mask)
# Pad seq_lens for CUDA graphs.
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(eagle_seq_lens_ptr + block, 0, mask=mask)
# Pad last_token_indices for CUDA graphs.
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(last_token_indices_ptr + block, 0, mask=mask)
def prepare_eagle_inputs(
# [num_reqs]
last_token_indices: torch.Tensor,
current_draft_step: torch.Tensor,
input_buffers: InputBuffers,
input_batch: InputBatch,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
max_num_reqs,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
_prepare_eagle_inputs_kernel[(num_reqs,)](
last_token_indices,
current_draft_step,
input_buffers.input_ids,
input_buffers.positions,
input_buffers.query_start_loc,
input_buffers.seq_lens,
input_batch.input_ids,
input_batch.positions,
input_batch.idx_mapping,
last_sampled,
next_prefill_tokens,
num_sampled,
num_rejected,
input_batch.query_start_loc,
input_batch.seq_lens,
max_num_reqs,
BLOCK_SIZE=1024,
)
return last_token_indices
@triton.jit
def _prepare_eagle_decode_kernel(
draft_tokens_ptr,
draft_tokens_stride,
target_seq_lens_ptr,
num_rejected_ptr,
input_ids_ptr,
positions_ptr,
query_start_loc_ptr,
seq_lens_ptr,
max_model_len,
max_num_reqs,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
num_reqs = tl.num_programs(0) - 1
if req_idx == num_reqs:
# Compute query_start_loc. Pad it with the last query_start_loc
# for CUDA graphs.
for i in range(0, max_num_reqs + 1, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
q = tl.where(block < num_reqs, block, num_reqs)
mask = block < max_num_reqs + 1
tl.store(query_start_loc_ptr + block, q, mask=mask)
# Pad seq_lens for CUDA graphs.
for i in range(req_idx, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(seq_lens_ptr + block, 0, mask=mask)
return
# draft token -> input id.
draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride)
tl.store(input_ids_ptr + req_idx, draft_token)
# Compute position and seq_lens.
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
# if they reach the max model length.
position = tl.load(positions_ptr + req_idx)
position = tl.minimum(position + 1, max_model_len - 1)
tl.store(positions_ptr + req_idx, position)
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
num_rejected = tl.load(num_rejected_ptr + req_idx)
seq_len = target_seq_len - num_rejected
seq_len = tl.minimum(seq_len + 1, max_model_len)
tl.store(seq_lens_ptr + req_idx, seq_len)
def prepare_eagle_decode(
draft_tokens: torch.Tensor,
target_seq_lens: torch.Tensor,
num_rejected: torch.Tensor,
input_buffers: InputBuffers,
max_model_len: int,
max_num_reqs: int,
):
num_reqs = draft_tokens.shape[0]
_prepare_eagle_decode_kernel[(num_reqs + 1,)](
draft_tokens,
draft_tokens.stride(0),
target_seq_lens,
num_rejected,
input_buffers.input_ids,
input_buffers.positions,
input_buffers.query_start_loc,
input_buffers.seq_lens,
max_model_len,
max_num_reqs,
BLOCK_SIZE=1024,
)
@triton.jit
def _update_eagle_draft_inputs_kernel(
output_draft_tokens_ptr,
output_draft_tokens_stride,
next_input_hidden_states_ptr,
next_input_hidden_states_stride,
input_ids_ptr,
positions_ptr,
seq_lens_ptr,
draft_tokens_ptr,
current_draft_step_ptr,
hidden_states_ptr,
hidden_states_stride,
hidden_size,
max_model_len,
num_speculative_steps,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
# Write the sampled draft token into self.draft_tokens[req_idx, step].
draft_token = tl.load(draft_tokens_ptr + req_idx)
step = tl.load(current_draft_step_ptr)
tl.store(
output_draft_tokens_ptr + req_idx * output_draft_tokens_stride + step,
draft_token,
)
if step >= num_speculative_steps - 1:
# This is the final step. Skip updating draft forward inputs.
return
# Write the sampled draft token into the input ids tensor for the next
# forward pass.
tl.store(input_ids_ptr + req_idx, draft_token)
# Copy hidden states into the input hidden states tensor for the next
# forward pass.
for i in range(0, hidden_size, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < hidden_size
hidden_states = tl.load(
hidden_states_ptr + req_idx * hidden_states_stride + block,
mask=mask,
)
tl.store(
next_input_hidden_states_ptr
+ req_idx * next_input_hidden_states_stride
+ block,
hidden_states,
mask=mask,
)
# Increment position and seq_lens.
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
# if they reach the max model length.
position = tl.load(positions_ptr + req_idx)
position = tl.minimum(position + 1, max_model_len - 1)
tl.store(positions_ptr + req_idx, position)
seq_len = tl.load(seq_lens_ptr + req_idx)
seq_len = tl.minimum(seq_len + 1, max_model_len)
tl.store(seq_lens_ptr + req_idx, seq_len)
def update_eagle_draft_inputs(
draft_tokens: torch.Tensor,
current_draft_step: torch.Tensor,
hidden_states: torch.Tensor,
output_draft_tokens: torch.Tensor,
next_input_hidden_states: torch.Tensor,
input_buffers: InputBuffers,
num_reqs: int,
max_model_len: int,
num_speculative_steps: int,
):
_, hidden_size = hidden_states.shape
_update_eagle_draft_inputs_kernel[(num_reqs,)](
output_draft_tokens,
output_draft_tokens.stride(0),
next_input_hidden_states,
next_input_hidden_states.stride(0),
input_buffers.input_ids,
input_buffers.positions,
input_buffers.seq_lens,
draft_tokens,
current_draft_step,
hidden_states,
hidden_states.stride(0),
hidden_size,
max_model_len,
num_speculative_steps,
BLOCK_SIZE=1024,
)
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
return load_eagle_model(target_model, self.vllm_config)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Gemma4 MTP (Multi-Token Prediction) speculator for speculative decoding.
The Gemma4 assistant model runs all decoder layers per draft step
(producing one token), and all its attention layers share KV cache
with the target model via cross-model KV sharing.
"""
from collections import defaultdict
import torch.nn as nn
from vllm.compilation.backends import set_model_tag
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
logger = init_logger(__name__)
class Gemma4Speculator(AutoRegressiveSpeculator):
@property
def advance_draft_positions(self) -> bool:
# Gemma4 MTP is Q-only and reads K/V from the target's existing cache.
# No new KV slots are written, so positions and seq_lens stay fixed.
return False
@property
def model_returns_tuple(self) -> bool:
# forward() returns (draft_hidden_states, backbone_hidden_states).
# The proposer uses draft_hidden_states for compute_logits and
# backbone_hidden_states for the hidden-state feedback buffer.
return True
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
draft_vllm_config = self._create_draft_vllm_config()
with set_model_tag("eagle_head"):
draft_model = get_model(
vllm_config=draft_vllm_config,
model_config=self.speculative_config.draft_model_config,
load_config=self.speculative_config.draft_load_config,
)
self._setup_gemma4_kv_sharing(draft_model, target_attn_layer_names)
self._share_embeddings(draft_model, target_model)
return draft_model
def _create_draft_vllm_config(self) -> VllmConfig:
"""Preserve the target's forced TRITON_ATTN backend for draft layers.
Gemma4 forces TRITON_ATTN due to heterogeneous head dimensions
(head_dim=256 sliding, global_head_dim=512 full). The base class
resets attention_config.backend to None for draft models, causing
sliding layers to fall back to FLASH_ATTN which cannot handle
KV-shared cache. Override to carry the target's backend through.
"""
draft_model_config = self.speculative_config.draft_model_config
draft_vllm_config = replace(
self.vllm_config,
model_config=draft_model_config,
)
target_backend = self.vllm_config.attention_config.backend
if target_backend is not None:
draft_vllm_config = replace(
draft_vllm_config,
attention_config=replace(
draft_vllm_config.attention_config,
backend=target_backend,
),
)
return draft_vllm_config
def _setup_gemma4_kv_sharing(
self,
model: nn.Module,
target_attn_layer_names: set[str],
) -> None:
"""Wire draft layers to share KV with the target model.
Each draft decoder layer is mapped to the last non-KV-shared
target layer of the same attention type (sliding or full).
"""
draft_config = self.speculative_config.draft_model_config.hf_config
draft_text_config = draft_config.get_text_config()
target_config = self.vllm_config.model_config.hf_config
target_text_config = target_config.get_text_config()
target_layer_types = getattr(target_text_config, "layer_types", [])
if not (hasattr(model, "model") and hasattr(model.model, "layers")):
return
target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0)
num_non_shared = len(target_layer_types) - target_num_kv_shared
type_to_target_indices: dict[str, list[int]] = defaultdict(list)
for idx, lt in enumerate(target_layer_types[:num_non_shared]):
type_to_target_indices[lt].append(idx)
target_prefix = "model.layers"
for name in target_attn_layer_names:
if ".layers." in name:
target_prefix = name.split(".layers.")[0] + ".layers"
break
draft_layer_types = getattr(draft_text_config, "layer_types", [])
for draft_idx, layer in enumerate(model.model.layers):
if not hasattr(layer, "self_attn"):
continue
attn = getattr(layer.self_attn, "attn", None)
if attn is None:
continue
draft_layer_type = (
draft_layer_types[draft_idx]
if draft_idx < len(draft_layer_types)
else "full_attention"
)
candidates = type_to_target_indices.get(draft_layer_type, [])
if not candidates:
logger.warning(
"No target layer of type '%s' for draft layer %d",
draft_layer_type,
draft_idx,
)
continue
target_idx = candidates[-1]
target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn"
attn.kv_sharing_target_layer_name = target_layer_name
logger.info(
"Gemma4 MTP: draft layer %d (%s) -> %s",
draft_idx,
draft_layer_type,
target_layer_name,
)
def _share_embeddings(
self,
draft_model: nn.Module,
target_model: nn.Module,
) -> None:
target_language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
if get_pp_group().world_size == 1:
target_embed = getattr(target_language_model.model, "embed_tokens", None)
if target_embed is not None:
del draft_model.model.embed_tokens
draft_model.model.embed_tokens = target_embed
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch.nn as nn
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
class MTPSpeculator(AutoRegressiveSpeculator):
@property
def model_returns_tuple(self) -> bool:
return False
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
return load_eagle_model(target_model, self.vllm_config)
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
init_attn_backend,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionStatePair,
BatchExecutionDescriptor,
)
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
class BaseSpeculator(ABC):
@abstractmethod
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
pass
@abstractmethod
def capture(
self,
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
) -> None:
pass
@abstractmethod
def propose(
self,
input_batch: InputBatch,
attn_metadata: dict[str, Any],
slot_mappings: dict[str, torch.Tensor],
# [num_tokens, hidden_size]
last_hidden_states: torch.Tensor,
# num_layers x [num_tokens, hidden_size]
aux_hidden_states: list[torch.Tensor] | None,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
pass
class DraftModelSpeculator(BaseSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
self.vllm_config = vllm_config
self.device = device
assert vllm_config.speculative_config is not None
self.speculative_config = vllm_config.speculative_config
self.method = self.speculative_config.method
self.num_speculative_steps = self.speculative_config.num_speculative_tokens
self.draft_model_config = self.speculative_config.draft_model_config
self.scheduler_config = vllm_config.scheduler_config
self.max_num_reqs = self.scheduler_config.max_num_seqs
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
self.max_model_len = vllm_config.model_config.max_model_len
self.draft_max_seq_len = self.max_model_len
# We need to get the hidden size from the draft model config because
# the draft model's hidden size can be different from the target model's
# hidden size (e.g., Llama 3.3 70B).
self.hidden_size = self.draft_model_config.get_hidden_size()
# Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP
# draft the target's pre-hc_head (T, hc_mult * hidden_size) residual).
# Non-HC models default to hc_mult=1 and are unaffected.
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
self.hidden_size = self.hidden_size * hc_mult
self.vocab_size = self.draft_model_config.get_vocab_size()
self.dtype = vllm_config.model_config.dtype
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
# DP configuration
self.dp_size = vllm_config.parallel_config.data_parallel_size
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
self.input_buffers = InputBuffers(
max_num_reqs=self.max_num_reqs,
max_num_tokens=self.max_num_tokens,
device=device,
)
self.idx_mapping = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=device
)
self.temperature = torch.zeros(
self.max_num_reqs, dtype=torch.float32, device=device
)
self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device)
self.draft_tokens = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps,
dtype=torch.int64,
device=device,
)
self.arange = torch.arange(
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
)
self.draft_logits: torch.Tensor | None = None
if self.speculative_config.draft_sample_method == "probabilistic":
self.draft_logits = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps,
self.vocab_size,
dtype=torch.float32,
device=device,
)
@abstractmethod
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
pass
def load_model(self, target_model: nn.Module) -> None:
target_attn_layer_names = set(
get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
).keys()
)
self.model = self.load_draft_model(target_model, target_attn_layer_names)
all_attn_layers = set[str](
get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
).keys()
)
self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names
def set_attn(
self,
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
) -> None:
self.model_state = model_state
self.kv_cache_config = kv_cache_config
self.attn_groups, _, _ = init_attn_backend(
kv_cache_config,
self.vllm_config,
self.device,
active_layer_names=self.draft_attn_layer_names,
)
self.block_tables = block_tables
def _build_draft_attn_metadata(
self,
num_reqs: int,
num_reqs_padded: int,
num_tokens_padded: int,
) -> dict[str, Any] | None:
query_start_loc_cpu = torch.clamp(
self.arange[: num_reqs_padded + 1], max=num_reqs
)
block_tables = [
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
]
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded]
attn_metadata = build_attn_metadata(
attn_groups=self.attn_groups,
num_reqs=num_reqs_padded,
num_tokens=num_tokens_padded,
query_start_loc_gpu=self.input_buffers.query_start_loc[
: num_reqs_padded + 1
],
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=1,
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
max_seq_len=self.draft_max_seq_len,
block_tables=block_tables,
slot_mappings=slot_mappings,
kv_cache_config=self.kv_cache_config,
)
return attn_metadata
def _copy_request_inputs(
self,
num_reqs: int,
# [num_reqs]
idx_mapping: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
) -> None:
# Copy temperature, seeds, and idx mapping to the pre-allocated buffers.
# NOTE(woosuk): For draft sampling, we only consider the temperature
# and ignore the other sampling parameters such as top_k and top_p,
# for simplicity and performance.
# While this may slightly degrade the acceptance rate, it does not
# affect the output distribution after rejection sampling.
self.temperature.copy_(temperature)
self.seeds.copy_(seeds)
self.idx_mapping[:num_reqs].copy_(idx_mapping)