[Model Runner V2][Spec Decode] Add Gemma4 MTP support (#43241)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -62,7 +62,6 @@ from vllm.v1.attention.backends.utils import (
|
||||
KVCacheLayoutType,
|
||||
get_dcp_local_seq_lens,
|
||||
get_kv_cache_layout,
|
||||
get_num_attention_heads_from_layers,
|
||||
get_per_layer_parameters,
|
||||
infer_global_hyperparameters,
|
||||
split_decodes_and_prefills,
|
||||
@@ -608,10 +607,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
)
|
||||
|
||||
# Compatible with models with non-uniform per-layer head counts.
|
||||
self.num_qo_heads = get_num_attention_heads_from_layers(
|
||||
vllm_config, layer_names
|
||||
) or self.model_config.get_num_attention_heads(self.vllm_config.parallel_config)
|
||||
self.num_qo_heads = self.model_config.get_num_attention_heads(
|
||||
self.vllm_config.parallel_config
|
||||
)
|
||||
|
||||
self.num_kv_heads = self.kv_cache_spec.num_kv_heads
|
||||
self.head_dim = self.kv_cache_spec.head_size
|
||||
|
||||
@@ -30,10 +30,7 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_kv_cache_layout,
|
||||
get_num_attention_heads_from_layers,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.attention.ops.triton_prefill_attention import context_attention_fwd
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash,
|
||||
@@ -142,10 +139,9 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
|
||||
self.block_size = kv_cache_spec.block_size
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
# Compatible with models with non-uniform per-layer head counts.
|
||||
self.num_heads_q = get_num_attention_heads_from_layers(
|
||||
vllm_config, layer_names
|
||||
) or model_config.get_num_attention_heads(vllm_config.parallel_config)
|
||||
self.num_heads_q = model_config.get_num_attention_heads(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
self.num_heads_kv = model_config.get_num_kv_heads(vllm_config.parallel_config)
|
||||
self.headdim = model_config.get_head_size()
|
||||
|
||||
|
||||
@@ -136,32 +136,6 @@ def get_per_layer_parameters(
|
||||
return per_layer_params
|
||||
|
||||
|
||||
def get_num_attention_heads_from_layers(
|
||||
vllm_config: VllmConfig, layer_names: list[str]
|
||||
) -> int | None:
|
||||
"""Per-TP-rank ``num_heads`` shared by the named Attention layers.
|
||||
|
||||
Use in metadata builders whose plan-time allocations depend on the
|
||||
head count: the model-wide ``get_num_attention_heads()`` is wrong
|
||||
for models with non-uniform per-layer head counts. All layers in
|
||||
one attention group must agree on ``num_heads``; this is asserted.
|
||||
Returns ``None`` when no matching Attention layer is found.
|
||||
"""
|
||||
attn_layers = get_layers_from_vllm_config(
|
||||
vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
layer_names,
|
||||
)
|
||||
if not attn_layers:
|
||||
return None
|
||||
heads = {layer.impl.num_heads for layer in attn_layers.values()}
|
||||
assert len(heads) == 1, (
|
||||
f"All layers in one attention group must share num_heads; "
|
||||
f"got {heads} for {layer_names}."
|
||||
)
|
||||
return heads.pop()
|
||||
|
||||
|
||||
def infer_global_hyperparameters(
|
||||
per_layer_params: dict[str, PerLayerParameters],
|
||||
) -> PerLayerParameters:
|
||||
|
||||
@@ -103,6 +103,7 @@ from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
set_eagle3_aux_hidden_state_layers,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||
@@ -307,7 +308,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.use_aux_hidden_state_outputs:
|
||||
assert self.speculative_config is not None
|
||||
set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config)
|
||||
if self.speculator is not None:
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
self.speculator.load_model(self.model)
|
||||
eplb_models_added = self.eplb.maybe_register_speculator(
|
||||
self.speculator, self.speculative_config, load_dummy_weights
|
||||
@@ -457,7 +458,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
|
||||
check_attention_cp_compatibility(self.vllm_config)
|
||||
if self.speculator is not None:
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
# HACK(woosuk)
|
||||
self.speculator.set_attn(
|
||||
self.model_state, self.kv_cache_config, self.block_tables
|
||||
|
||||
@@ -8,8 +8,21 @@ from vllm.config import VllmConfig
|
||||
def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
if speculative_config.use_eagle():
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator
|
||||
if speculative_config.use_gemma4_mtp():
|
||||
from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import (
|
||||
Gemma4Speculator,
|
||||
)
|
||||
|
||||
return Gemma4Speculator(vllm_config, device)
|
||||
elif speculative_config.method == "mtp":
|
||||
from vllm.v1.worker.gpu.spec_decode.mtp.speculator import MTPSpeculator
|
||||
|
||||
return MTPSpeculator(vllm_config, device)
|
||||
elif speculative_config.use_eagle():
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import (
|
||||
EagleSpeculator,
|
||||
)
|
||||
|
||||
return EagleSpeculator(vllm_config, device)
|
||||
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
|
||||
else:
|
||||
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
+4
-5
@@ -19,8 +19,8 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class PrefillEagleCudaGraphManager(CudaGraphManager):
|
||||
"""Eagle CudaGraphManager for prefill, using pre-built attention states
|
||||
class PrefillSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
"""CudaGraphManager for draft prefill, using pre-built attention states
|
||||
from the target model's capture."""
|
||||
|
||||
def capture(
|
||||
@@ -56,9 +56,8 @@ class PrefillEagleCudaGraphManager(CudaGraphManager):
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
|
||||
class DecodeEagleCudaGraphManager(CudaGraphManager):
|
||||
"""Eagle CudaGraphManager for decode draft generation, building its own
|
||||
attention metadata from scratch."""
|
||||
class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
"""CudaGraphManager for draft decode, building its own attention metadata."""
|
||||
|
||||
def capture(
|
||||
self,
|
||||
@@ -0,0 +1,795 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
DecodeSpeculatorCudaGraphManager,
|
||||
PrefillSpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
self.hidden_states = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device)
|
||||
self.last_token_indices = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
|
||||
self.draft_model_config
|
||||
)
|
||||
if self.supports_mm_inputs:
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.prefill_cudagraph_manager: PrefillSpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: DecodeSpeculatorCudaGraphManager | None = None
|
||||
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
"""
|
||||
Whether to increment positions and seq_lens between draft steps.
|
||||
|
||||
True for Eagle/standard MTP (each step produces new KV).
|
||||
False for Gemma4 MTP (Q-only, shares target KV, constant positions).
|
||||
"""
|
||||
return True
|
||||
|
||||
@property
|
||||
def model_returns_tuple(self) -> bool:
|
||||
"""
|
||||
Whether the draft model's forward() returns a tuple.
|
||||
|
||||
True: returns (last_hidden_states, hidden_states) — Eagle, Gemma4 MTP.
|
||||
False: returns a single tensor used for both — standard MTP (DeepSeek).
|
||||
"""
|
||||
return True
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# Initialize cudagraph manager for draft prefill (draft position 0).
|
||||
self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
self.num_speculative_steps + 1,
|
||||
)
|
||||
|
||||
# PIECEWISE cudagraphs are not supported for draft decodes.
|
||||
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
|
||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
else:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# Initialize cudagraph manager for draft decodes (draft positions > 0).
|
||||
self.decode_cudagraph_manager = DecodeSpeculatorCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=1,
|
||||
)
|
||||
|
||||
def capture(
|
||||
self,
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
) -> None:
|
||||
logger.info("Capturing model for speculator...")
|
||||
# Reset indices to zeros to prevent stale values from prior
|
||||
# dummy runs to cause out-of-bounds indexing during capture.
|
||||
self.last_token_indices.zero_()
|
||||
|
||||
# Capture the prefill routine (model forward + compute_logits +
|
||||
# sample).
|
||||
# For FULL graphs, the entire routine is recorded as one graph.
|
||||
# For PIECEWISE, only the model's compiled regions are captured
|
||||
# and the rest (compute_logits, gumbel_sample) runs eagerly.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
if self.prefill_cudagraph_manager.use_breakable_cg:
|
||||
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
|
||||
self.prefill_cudagraph_manager.capture(
|
||||
self._prefill,
|
||||
attn_states,
|
||||
progress_bar_desc="Capturing prefill CUDA graphs",
|
||||
)
|
||||
|
||||
if self.num_speculative_steps == 1:
|
||||
return
|
||||
|
||||
# Capture the decode draft generation routine (model forward +
|
||||
# sample + update_draft_inputs) for a single
|
||||
# step.
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.capture(
|
||||
self._generate_draft,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
progress_bar_desc="Capturing decode CUDA graphs",
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
attn_metadata: dict[str, Any],
|
||||
slot_mappings: dict[str, torch.Tensor],
|
||||
# [num_tokens, hidden_size]
|
||||
last_hidden_states: torch.Tensor,
|
||||
# num_layers x [num_tokens, hidden_size]
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
num_reqs = input_batch.num_reqs
|
||||
max_query_len = input_batch.num_scheduled_tokens.max()
|
||||
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
||||
self.draft_max_seq_len = min(
|
||||
max_seq_len + self.num_speculative_steps, self.max_model_len
|
||||
)
|
||||
|
||||
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
|
||||
# number of rejected tokens, we maintain the size of input_ids and
|
||||
# hidden_states the same as the target model's. This means, we pad each
|
||||
# request's query length to include any rejected positions. By doing so,
|
||||
# we can also reuse the attention metadata (e.g., query_start_loc,
|
||||
# seq_lens) of the target model.
|
||||
if aux_hidden_states:
|
||||
assert self.method == "eagle3"
|
||||
hidden_states = self.model.combine_hidden_states(
|
||||
torch.cat(aux_hidden_states, dim=-1)
|
||||
)
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_tokens].copy_(hidden_states)
|
||||
|
||||
self._copy_request_inputs(
|
||||
num_reqs,
|
||||
input_batch.idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
)
|
||||
|
||||
# Get the input ids and last token indices for the speculator.
|
||||
prepare_prefill_inputs(
|
||||
self.last_token_indices,
|
||||
self.current_draft_step,
|
||||
self.input_buffers,
|
||||
input_batch,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
self.max_num_reqs,
|
||||
)
|
||||
|
||||
# When all requests are decoding (no true prefills), each has
|
||||
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
|
||||
uniform_token_count = get_uniform_token_count(
|
||||
num_reqs,
|
||||
# Use the actual number of tokens without padding added by
|
||||
# the target model during FULL cudagraph.
|
||||
input_batch.num_tokens,
|
||||
max_query_len,
|
||||
)
|
||||
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.prefill_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
uniform_token_count,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Replay the full graph for draft prefill.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
self.prefill_cudagraph_manager.run_fullgraph(prefill_batch_desc)
|
||||
else:
|
||||
# The target model's attention metadata and slot mappings
|
||||
# can directly be used for draft prefill, because of the
|
||||
# identical batch shape and KV cache layout.
|
||||
self._prefill(
|
||||
num_reqs,
|
||||
prefill_batch_desc.num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=prefill_batch_desc.cg_mode,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
|
||||
if self.num_speculative_steps == 1:
|
||||
# Early exit.
|
||||
return self.draft_tokens[:num_reqs, :1]
|
||||
|
||||
# Prepare the inputs for the decode steps.
|
||||
prepare_decode_inputs(
|
||||
self.draft_tokens[:num_reqs, 0],
|
||||
input_batch.seq_lens,
|
||||
num_rejected,
|
||||
self.input_buffers,
|
||||
self.max_model_len,
|
||||
self.max_num_reqs,
|
||||
advance_draft_positions=self.advance_draft_positions,
|
||||
)
|
||||
|
||||
# Each request produces exactly 1 token per draft generation step,
|
||||
# enabling FULL graph replay.
|
||||
decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.decode_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_reqs,
|
||||
uniform_token_count=1,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
# Generate the remaining num_speculative_steps - 1 draft tokens.
|
||||
self._multi_step_decode(
|
||||
num_reqs,
|
||||
dummy_run and skip_attn_for_dummy_run,
|
||||
decode_batch_desc,
|
||||
num_tokens_across_dp,
|
||||
)
|
||||
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
def sample_draft(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
if draft_logits is not None:
|
||||
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
|
||||
# used for draft and target sampling.
|
||||
return gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
positions + 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=draft_logits,
|
||||
output_processed_logits_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(
|
||||
self,
|
||||
num_tokens: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
|
||||
with set_forward_context(
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
):
|
||||
inputs_embeds = None
|
||||
if self.supports_mm_inputs:
|
||||
# Merge multimodal embeddings with input ids.
|
||||
mm_embeds, is_mm_embed = mm_inputs or (None, None)
|
||||
num_input_tokens = (
|
||||
is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens
|
||||
)
|
||||
self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids(
|
||||
self.input_buffers.input_ids[:num_input_tokens],
|
||||
multimodal_embeddings=mm_embeds,
|
||||
is_multimodal=is_mm_embed,
|
||||
)
|
||||
inputs_embeds = self.inputs_embeds[:num_tokens]
|
||||
|
||||
model_inputs = dict(
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens],
|
||||
positions=self.input_buffers.positions[:num_tokens],
|
||||
hidden_states=self.hidden_states[:num_tokens],
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
|
||||
# Draft prefill with PIECEWISE cudagraph (compiled PW or breakable),
|
||||
# chosen inside run_pw_graph.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph(
|
||||
self.model, model_inputs
|
||||
)
|
||||
else:
|
||||
# Eager (NONE): call the raw model directly.
|
||||
ret_hidden_states = self.model(**model_inputs)
|
||||
if self.model_returns_tuple:
|
||||
last_hidden_states, hidden_states = ret_hidden_states
|
||||
else:
|
||||
last_hidden_states = ret_hidden_states
|
||||
hidden_states = ret_hidden_states
|
||||
return last_hidden_states, hidden_states
|
||||
|
||||
def _prefill(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
last_token_indices = self.last_token_indices[:num_reqs]
|
||||
positions = self.input_buffers.positions[last_token_indices]
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
last_hidden_states, hidden_states = self._run_model(
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
sample_hidden_states = last_hidden_states[last_token_indices]
|
||||
|
||||
self.draft_tokens[:num_reqs, 0] = self.sample_draft(
|
||||
sample_hidden_states,
|
||||
positions,
|
||||
idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.current_draft_step,
|
||||
self.draft_logits,
|
||||
)
|
||||
self.hidden_states[:num_reqs] = hidden_states[last_token_indices]
|
||||
self.input_buffers.positions[:num_reqs] = positions
|
||||
|
||||
def _multi_step_decode(
|
||||
self,
|
||||
num_reqs: int,
|
||||
skip_attn: bool,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
) -> None:
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
for step in range(1, self.num_speculative_steps):
|
||||
# Rebuild every step when positions advance, or just once
|
||||
# on the first step when positions are constant (Gemma4 MTP).
|
||||
if not skip_attn and (self.advance_draft_positions or step == 1):
|
||||
slot_mappings = self.block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
batch_desc.num_tokens,
|
||||
)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
attn_metadata = self._build_draft_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=batch_desc.num_reqs or num_reqs,
|
||||
num_tokens_padded=batch_desc.num_tokens,
|
||||
)
|
||||
|
||||
# Update the current draft step.
|
||||
self.current_draft_step.fill_(step)
|
||||
|
||||
# Generate draft tokens for the current step.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
batch_desc.num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings_by_layer,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
# Run the draft model forward pass.
|
||||
last_hidden_states, hidden_states = self._run_model(
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
last_hidden_states = last_hidden_states[:num_reqs]
|
||||
|
||||
# Sample the draft tokens.
|
||||
draft_tokens = self.sample_draft(
|
||||
last_hidden_states,
|
||||
positions,
|
||||
idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.current_draft_step,
|
||||
self.draft_logits,
|
||||
)
|
||||
|
||||
# Update the inputs for the next step.
|
||||
update_draft_inputs(
|
||||
draft_tokens,
|
||||
self.current_draft_step,
|
||||
hidden_states,
|
||||
self.draft_tokens,
|
||||
self.hidden_states,
|
||||
self.input_buffers,
|
||||
num_reqs,
|
||||
self.max_model_len,
|
||||
self.num_speculative_steps,
|
||||
advance_draft_positions=self.advance_draft_positions,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_prefill_inputs_kernel(
|
||||
last_token_indices_ptr,
|
||||
draft_current_step_ptr,
|
||||
draft_input_ids_ptr,
|
||||
draft_positions_ptr,
|
||||
draft_query_start_loc_ptr,
|
||||
draft_seq_lens_ptr,
|
||||
target_input_ids_ptr,
|
||||
target_positions_ptr,
|
||||
idx_mapping_ptr,
|
||||
last_sampled_ptr,
|
||||
next_prefill_tokens_ptr,
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
num_reqs = tl.num_programs(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + req_idx)
|
||||
query_end = tl.load(query_start_loc_ptr + req_idx + 1)
|
||||
query_len = query_end - query_start
|
||||
seq_len = tl.load(seq_lens_ptr + req_idx)
|
||||
|
||||
# Get the true query length and next token after accounting for rejected tokens.
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
query_len -= num_rejected
|
||||
|
||||
num_sampled = tl.load(num_sampled_ptr + req_idx)
|
||||
if num_sampled > 0:
|
||||
next_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32)
|
||||
else:
|
||||
# Chunked prefilling.
|
||||
# Get the next prefill token.
|
||||
next_token = tl.load(next_prefill_tokens_ptr + req_state_idx)
|
||||
|
||||
# Shift target_input_ids by one.
|
||||
for i in range(1, query_len, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < query_len
|
||||
input_ids = tl.load(target_input_ids_ptr + query_start + block, mask=mask)
|
||||
tl.store(draft_input_ids_ptr + query_start + block - 1, input_ids, mask=mask)
|
||||
|
||||
last_token_index = query_start + query_len - 1
|
||||
tl.store(last_token_indices_ptr + req_idx, last_token_index)
|
||||
tl.store(draft_input_ids_ptr + last_token_index, next_token)
|
||||
|
||||
# Copy positions.
|
||||
for i in range(0, query_len, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < query_len
|
||||
target_pos = tl.load(target_positions_ptr + query_start + block, mask=mask)
|
||||
tl.store(draft_positions_ptr + query_start + block, target_pos, mask=mask)
|
||||
|
||||
# Copy query start locations.
|
||||
tl.store(draft_query_start_loc_ptr + req_idx, query_start)
|
||||
# Copy sequence lengths.
|
||||
tl.store(draft_seq_lens_ptr + req_idx, seq_len)
|
||||
if req_idx == (num_reqs - 1):
|
||||
# Reset the current draft step to 0.
|
||||
tl.store(draft_current_step_ptr, 0)
|
||||
# Pad query_start_loc for CUDA graphs.
|
||||
for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs + 1
|
||||
tl.store(draft_query_start_loc_ptr + block, query_end, mask=mask)
|
||||
# Pad seq_lens for CUDA graphs.
|
||||
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(draft_seq_lens_ptr + block, 0, mask=mask)
|
||||
# Pad last_token_indices for CUDA graphs.
|
||||
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(last_token_indices_ptr + block, 0, mask=mask)
|
||||
|
||||
|
||||
def prepare_prefill_inputs(
|
||||
# [num_reqs]
|
||||
last_token_indices: torch.Tensor,
|
||||
current_draft_step: torch.Tensor,
|
||||
input_buffers: InputBuffers,
|
||||
input_batch: InputBatch,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
max_num_reqs,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
_prepare_prefill_inputs_kernel[(num_reqs,)](
|
||||
last_token_indices,
|
||||
current_draft_step,
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
input_batch.input_ids,
|
||||
input_batch.positions,
|
||||
input_batch.idx_mapping,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.seq_lens,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
return last_token_indices
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_decode_inputs_kernel(
|
||||
draft_tokens_ptr,
|
||||
draft_tokens_stride,
|
||||
target_seq_lens_ptr,
|
||||
num_rejected_ptr,
|
||||
input_ids_ptr,
|
||||
positions_ptr,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
max_model_len,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
ADVANCE_DRAFT_POSITIONS: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
num_reqs = tl.num_programs(0) - 1
|
||||
if req_idx == num_reqs:
|
||||
# Compute query_start_loc. Pad it with the last query_start_loc
|
||||
# for CUDA graphs.
|
||||
for i in range(0, max_num_reqs + 1, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
q = tl.where(block < num_reqs, block, num_reqs)
|
||||
mask = block < max_num_reqs + 1
|
||||
tl.store(query_start_loc_ptr + block, q, mask=mask)
|
||||
# Pad seq_lens for CUDA graphs.
|
||||
for i in range(req_idx, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(seq_lens_ptr + block, 0, mask=mask)
|
||||
return
|
||||
|
||||
# draft token -> input id.
|
||||
draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride)
|
||||
tl.store(input_ids_ptr + req_idx, draft_token)
|
||||
|
||||
if ADVANCE_DRAFT_POSITIONS:
|
||||
# Compute position and seq_lens.
|
||||
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
|
||||
# if they reach the max model length.
|
||||
position = tl.load(positions_ptr + req_idx)
|
||||
position = tl.minimum(position + 1, max_model_len - 1)
|
||||
tl.store(positions_ptr + req_idx, position)
|
||||
|
||||
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
seq_len = target_seq_len - num_rejected
|
||||
seq_len = tl.minimum(seq_len + 1, max_model_len)
|
||||
tl.store(seq_lens_ptr + req_idx, seq_len)
|
||||
|
||||
|
||||
def prepare_decode_inputs(
|
||||
draft_tokens: torch.Tensor,
|
||||
target_seq_lens: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
input_buffers: InputBuffers,
|
||||
max_model_len: int,
|
||||
max_num_reqs: int,
|
||||
advance_draft_positions: bool = True,
|
||||
):
|
||||
num_reqs = draft_tokens.shape[0]
|
||||
_prepare_decode_inputs_kernel[(num_reqs + 1,)](
|
||||
draft_tokens,
|
||||
draft_tokens.stride(0),
|
||||
target_seq_lens,
|
||||
num_rejected,
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
max_model_len,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE=1024,
|
||||
ADVANCE_DRAFT_POSITIONS=advance_draft_positions,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _update_draft_inputs_kernel(
|
||||
output_draft_tokens_ptr,
|
||||
output_draft_tokens_stride,
|
||||
next_input_hidden_states_ptr,
|
||||
next_input_hidden_states_stride,
|
||||
input_ids_ptr,
|
||||
positions_ptr,
|
||||
seq_lens_ptr,
|
||||
draft_tokens_ptr,
|
||||
current_draft_step_ptr,
|
||||
hidden_states_ptr,
|
||||
hidden_states_stride,
|
||||
hidden_size,
|
||||
max_model_len,
|
||||
num_speculative_steps,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
ADVANCE_DRAFT_POSITIONS: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
|
||||
# Write the sampled draft token into self.draft_tokens[req_idx, step].
|
||||
draft_token = tl.load(draft_tokens_ptr + req_idx)
|
||||
step = tl.load(current_draft_step_ptr)
|
||||
tl.store(
|
||||
output_draft_tokens_ptr + req_idx * output_draft_tokens_stride + step,
|
||||
draft_token,
|
||||
)
|
||||
|
||||
if step >= num_speculative_steps - 1:
|
||||
# This is the final step. Skip updating draft forward inputs.
|
||||
return
|
||||
|
||||
# Write the sampled draft token into the input ids tensor for the next
|
||||
# forward pass.
|
||||
tl.store(input_ids_ptr + req_idx, draft_token)
|
||||
|
||||
# Copy hidden states into the input hidden states tensor for the next
|
||||
# forward pass.
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < hidden_size
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + req_idx * hidden_states_stride + block,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
next_input_hidden_states_ptr
|
||||
+ req_idx * next_input_hidden_states_stride
|
||||
+ block,
|
||||
hidden_states,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
if ADVANCE_DRAFT_POSITIONS:
|
||||
# Increment position and seq_lens.
|
||||
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
|
||||
# if they reach the max model length.
|
||||
position = tl.load(positions_ptr + req_idx)
|
||||
position = tl.minimum(position + 1, max_model_len - 1)
|
||||
tl.store(positions_ptr + req_idx, position)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + req_idx)
|
||||
seq_len = tl.minimum(seq_len + 1, max_model_len)
|
||||
tl.store(seq_lens_ptr + req_idx, seq_len)
|
||||
|
||||
|
||||
def update_draft_inputs(
|
||||
draft_tokens: torch.Tensor,
|
||||
current_draft_step: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
output_draft_tokens: torch.Tensor,
|
||||
next_input_hidden_states: torch.Tensor,
|
||||
input_buffers: InputBuffers,
|
||||
num_reqs: int,
|
||||
max_model_len: int,
|
||||
num_speculative_steps: int,
|
||||
advance_draft_positions: bool = True,
|
||||
):
|
||||
_, hidden_size = hidden_states.shape
|
||||
_update_draft_inputs_kernel[(num_reqs,)](
|
||||
output_draft_tokens,
|
||||
output_draft_tokens.stride(0),
|
||||
next_input_hidden_states,
|
||||
next_input_hidden_states.stride(0),
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.seq_lens,
|
||||
draft_tokens,
|
||||
current_draft_step,
|
||||
hidden_states,
|
||||
hidden_states.stride(0),
|
||||
hidden_size,
|
||||
max_model_len,
|
||||
num_speculative_steps,
|
||||
BLOCK_SIZE=1024,
|
||||
ADVANCE_DRAFT_POSITIONS=advance_draft_positions,
|
||||
)
|
||||
@@ -1,903 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
build_slot_mappings_by_layer,
|
||||
init_attn_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.cudagraph import (
|
||||
DecodeEagleCudaGraphManager,
|
||||
PrefillEagleCudaGraphManager,
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class EagleSpeculator:
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
assert self.speculative_config is not None
|
||||
self.method = self.speculative_config.method
|
||||
self.num_speculative_steps = self.speculative_config.num_speculative_tokens
|
||||
self.draft_model_config = self.speculative_config.draft_model_config
|
||||
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.max_num_reqs = self.scheduler_config.max_num_seqs
|
||||
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.draft_max_seq_len = self.max_model_len
|
||||
# We need to get the hidden size from the draft model config because
|
||||
# the draft model's hidden size can be different from the target model's
|
||||
# hidden size (e.g., Llama 3.3 70B).
|
||||
self.hidden_size = self.draft_model_config.get_hidden_size()
|
||||
# Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP
|
||||
# draft the target's pre-hc_head (T, hc_mult * hidden_size) residual).
|
||||
# Non-HC models default to hc_mult=1 and are unaffected.
|
||||
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
|
||||
self.hidden_size = self.hidden_size * hc_mult
|
||||
self.vocab_size = self.draft_model_config.get_vocab_size()
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
|
||||
|
||||
# DP configuration
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
device=device,
|
||||
)
|
||||
self.hidden_states = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
self.idx_mapping = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self.temperature = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.float32, device=device
|
||||
)
|
||||
self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
self.draft_tokens = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device)
|
||||
self.last_token_indices = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int64, device=device
|
||||
)
|
||||
self.arange = torch.arange(
|
||||
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
|
||||
self.draft_model_config
|
||||
)
|
||||
if self.supports_mm_inputs:
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
self.draft_logits = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.vocab_size,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
self.prefill_cudagraph_manager: PrefillEagleCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: DecodeEagleCudaGraphManager | None = None
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
cudagraph_mode = self.vllm_config.compilation_config.cudagraph_mode
|
||||
# Initialize cudagraph manager for draft prefill (draft position 0).
|
||||
self.prefill_cudagraph_manager = PrefillEagleCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
self.num_speculative_steps + 1,
|
||||
)
|
||||
|
||||
# PIECEWISE cudagraphs are not supported for eagle draft decodes.
|
||||
# PIECEWISE pads num_tokens to the next capture size without padding
|
||||
# num_reqs, which can cause attention backends to read past the
|
||||
# valid per-request metadata (e.g. FlashInfer's kv_indptr buffer).
|
||||
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
|
||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
else:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# Initialize cudagraph manager for draft decodes (draft positions > 0).
|
||||
self.decode_cudagraph_manager = DecodeEagleCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=1,
|
||||
)
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
target_attn_layer_names = get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
).keys()
|
||||
|
||||
self.model = load_eagle_model(target_model, self.vllm_config)
|
||||
|
||||
all_attn_layers = get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
).keys()
|
||||
self.draft_attn_layer_names = set(all_attn_layers) - set(
|
||||
target_attn_layer_names
|
||||
)
|
||||
|
||||
def set_attn(
|
||||
class EagleSpeculator(AutoRegressiveSpeculator):
|
||||
def load_draft_model(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
) -> None:
|
||||
self.model_state = model_state
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.attn_groups, _, _ = init_attn_backend(
|
||||
kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
active_layer_names=self.draft_attn_layer_names,
|
||||
)
|
||||
self.block_tables = block_tables
|
||||
|
||||
@torch.inference_mode()
|
||||
def run_model(
|
||||
self,
|
||||
num_tokens: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
|
||||
with set_forward_context(
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
):
|
||||
inputs_embeds = None
|
||||
if self.supports_mm_inputs:
|
||||
# Merge multimodal embeddings with input ids.
|
||||
mm_embeds, is_mm_embed = mm_inputs or (None, None)
|
||||
num_input_tokens = (
|
||||
is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens
|
||||
)
|
||||
self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids(
|
||||
self.input_buffers.input_ids[:num_input_tokens],
|
||||
multimodal_embeddings=mm_embeds,
|
||||
is_multimodal=is_mm_embed,
|
||||
)
|
||||
inputs_embeds = self.inputs_embeds[:num_tokens]
|
||||
|
||||
model_inputs = dict(
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens],
|
||||
positions=self.input_buffers.positions[:num_tokens],
|
||||
hidden_states=self.hidden_states[:num_tokens],
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
|
||||
# Draft prefill with PIECEWISE cudagraph (compiled PW or breakable),
|
||||
# chosen inside run_pw_graph.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph(
|
||||
self.model, model_inputs
|
||||
)
|
||||
else:
|
||||
# Eager (NONE): call the raw model directly.
|
||||
ret_hidden_states = self.model(**model_inputs)
|
||||
if self.method == "mtp":
|
||||
last_hidden_states = ret_hidden_states
|
||||
hidden_states = ret_hidden_states
|
||||
else:
|
||||
last_hidden_states, hidden_states = ret_hidden_states
|
||||
return last_hidden_states, hidden_states
|
||||
|
||||
def _sample_draft(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if draft_logits is not None:
|
||||
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
|
||||
# used for draft and target sampling.
|
||||
return gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
pos + 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=draft_logits,
|
||||
output_processed_logits_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
def prefill(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
last_token_indices = self.last_token_indices[:num_reqs]
|
||||
pos = self.input_buffers.positions[last_token_indices]
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
last_hidden_states, hidden_states = self.run_model(
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
sample_hidden_states = last_hidden_states[last_token_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
self.draft_tokens[:num_reqs, 0] = self._sample_draft(
|
||||
logits,
|
||||
idx_mapping,
|
||||
pos,
|
||||
self.current_draft_step,
|
||||
self.draft_logits,
|
||||
)
|
||||
self.hidden_states[:num_reqs] = hidden_states[last_token_indices]
|
||||
self.input_buffers.positions[:num_reqs] = pos
|
||||
|
||||
def multi_step_decode(
|
||||
self,
|
||||
num_reqs: int,
|
||||
skip_attn: bool,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
) -> None:
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
for step in range(1, self.num_speculative_steps):
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not skip_attn:
|
||||
# Build attention metadata and slot mappings for each draft
|
||||
# decode step. It is necessary to rebuild the attention
|
||||
# metadata even when replaying the FULL graph so that any
|
||||
# attention metadata builder state is updated.
|
||||
slot_mappings = self.block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
batch_desc.num_tokens,
|
||||
)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
attn_metadata = self._build_draft_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=batch_desc.num_reqs or num_reqs,
|
||||
num_tokens_padded=batch_desc.num_tokens,
|
||||
)
|
||||
|
||||
# Update the current draft step.
|
||||
self.current_draft_step.fill_(step)
|
||||
|
||||
# Generate draft tokens for the current step.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
self.generate_draft(
|
||||
num_reqs,
|
||||
batch_desc.num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings_by_layer,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
)
|
||||
|
||||
def generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
# Run the eagle model forward pass.
|
||||
last_hidden_states, hidden_states = self.run_model(
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
last_hidden_states = last_hidden_states[:num_reqs]
|
||||
|
||||
# Sample the draft tokens.
|
||||
logits = self.model.compute_logits(last_hidden_states)
|
||||
draft_tokens = self._sample_draft(
|
||||
logits,
|
||||
idx_mapping,
|
||||
positions,
|
||||
self.current_draft_step,
|
||||
self.draft_logits,
|
||||
)
|
||||
|
||||
# Update the inputs for the next step.
|
||||
update_eagle_draft_inputs(
|
||||
draft_tokens,
|
||||
self.current_draft_step,
|
||||
hidden_states,
|
||||
self.draft_tokens,
|
||||
self.hidden_states,
|
||||
self.input_buffers,
|
||||
num_reqs,
|
||||
self.max_model_len,
|
||||
self.num_speculative_steps,
|
||||
)
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
|
||||
query_start_loc_cpu = torch.clamp(
|
||||
self.arange[: num_reqs_padded + 1], max=num_reqs
|
||||
)
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
]
|
||||
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded]
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=self.attn_groups,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_tokens=num_tokens_padded,
|
||||
query_start_loc_gpu=self.input_buffers.query_start_loc[
|
||||
: num_reqs_padded + 1
|
||||
],
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=1,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
max_seq_len=self.draft_max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def capture(
|
||||
self,
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
) -> None:
|
||||
logger.info("Capturing model for Eagle speculator...")
|
||||
# Reset indices to zeros to prevent stale values from prior
|
||||
# dummy runs to cause out-of-bounds indexing during capture.
|
||||
self.last_token_indices.zero_()
|
||||
|
||||
# Capture the prefill routine (model forward + compute_logits +
|
||||
# sample).
|
||||
# For FULL graphs, the entire routine is recorded as one graph.
|
||||
# For PIECEWISE, only the model's compiled regions are captured
|
||||
# and the rest (compute_logits, gumbel_sample) runs eagerly.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
if self.prefill_cudagraph_manager.use_breakable_cg:
|
||||
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
|
||||
self.prefill_cudagraph_manager.capture(
|
||||
self.prefill,
|
||||
attn_states,
|
||||
progress_bar_desc="Capturing eagle prefill CUDA graphs",
|
||||
)
|
||||
|
||||
if self.num_speculative_steps == 1:
|
||||
return
|
||||
|
||||
# Capture the decode draft generation routine (model forward +
|
||||
# compute_logits + sample + update_eagle_inputs) for a single
|
||||
# step.
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.capture(
|
||||
self.generate_draft,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
progress_bar_desc="Capturing eagle decode CUDA graphs",
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
attn_metadata: dict[str, Any],
|
||||
slot_mappings: dict[str, torch.Tensor],
|
||||
# [num_tokens, hidden_size]
|
||||
last_hidden_states: torch.Tensor,
|
||||
# num_layers x [num_tokens, hidden_size]
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
num_reqs = input_batch.num_reqs
|
||||
max_query_len = input_batch.num_scheduled_tokens.max()
|
||||
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
||||
self.draft_max_seq_len = min(
|
||||
max_seq_len + self.num_speculative_steps, self.max_model_len
|
||||
)
|
||||
|
||||
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
|
||||
# number of rejected tokens, we maintain the size of eagle's input_ids and
|
||||
# hidden_states the same as the target model's. This means, we pad each
|
||||
# request's query length to include any rejected positions. By doing so,
|
||||
# we can also reuse the attention metadata (e.g., query_start_loc,
|
||||
# seq_lens) of the target model.
|
||||
if aux_hidden_states:
|
||||
assert self.method == "eagle3"
|
||||
hidden_states = self.model.combine_hidden_states(
|
||||
torch.cat(aux_hidden_states, dim=-1)
|
||||
)
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_tokens].copy_(hidden_states)
|
||||
|
||||
# Copy temperature, seeds, and idx mapping to the pre-allocated buffers.
|
||||
# NOTE(woosuk): For draft sampling, we only consider the temperature
|
||||
# and ignore the other sampling parameters such as top_k and top_p,
|
||||
# for simplicity and performance.
|
||||
# While this may slightly degrade the acceptance rate, it does not
|
||||
# affect the output distribution after rejection sampling.
|
||||
self.temperature.copy_(temperature)
|
||||
self.seeds.copy_(seeds)
|
||||
self.idx_mapping[:num_reqs].copy_(input_batch.idx_mapping)
|
||||
|
||||
# Get the input ids and last token indices for the speculator.
|
||||
prepare_eagle_inputs(
|
||||
self.last_token_indices,
|
||||
self.current_draft_step,
|
||||
self.input_buffers,
|
||||
input_batch,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
self.max_num_reqs,
|
||||
)
|
||||
|
||||
# When all requests are decoding (no true prefills), each has
|
||||
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
|
||||
uniform_token_count = get_uniform_token_count(
|
||||
num_reqs,
|
||||
# Use the actual number of tokens without padding added by
|
||||
# the target model during FULL cudagraph.
|
||||
input_batch.num_tokens,
|
||||
max_query_len,
|
||||
)
|
||||
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.prefill_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
uniform_token_count,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Replay the full graph for draft prefill.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
self.prefill_cudagraph_manager.run_fullgraph(prefill_batch_desc)
|
||||
else:
|
||||
# The target model's attention metadata and slot mappings
|
||||
# can directly be used for draft prefill, because of the
|
||||
# identical batch shape and KV cache layout.
|
||||
self.prefill(
|
||||
num_reqs,
|
||||
prefill_batch_desc.num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=prefill_batch_desc.cg_mode,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
|
||||
if self.num_speculative_steps == 1:
|
||||
# Early exit.
|
||||
return self.draft_tokens[:num_reqs, :1]
|
||||
|
||||
# Prepare the inputs for the decode steps.
|
||||
prepare_eagle_decode(
|
||||
self.draft_tokens[:num_reqs, 0],
|
||||
input_batch.seq_lens,
|
||||
num_rejected,
|
||||
self.input_buffers,
|
||||
self.max_model_len,
|
||||
self.max_num_reqs,
|
||||
)
|
||||
|
||||
# Each request produces exactly 1 token per draft generation step,
|
||||
# enabling FULL graph replay.
|
||||
decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.decode_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_reqs,
|
||||
uniform_token_count=1,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
# Generate the remaining num_speculative_steps - 1 draft tokens.
|
||||
self.multi_step_decode(
|
||||
num_reqs,
|
||||
dummy_run and skip_attn_for_dummy_run,
|
||||
decode_batch_desc,
|
||||
num_tokens_across_dp,
|
||||
)
|
||||
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_eagle_inputs_kernel(
|
||||
last_token_indices_ptr,
|
||||
eagle_current_draft_step_ptr,
|
||||
eagle_input_ids_ptr,
|
||||
eagle_positions_ptr,
|
||||
eagle_query_start_loc_ptr,
|
||||
eagle_seq_lens_ptr,
|
||||
target_input_ids_ptr,
|
||||
target_positions_ptr,
|
||||
idx_mapping_ptr,
|
||||
last_sampled_ptr,
|
||||
next_prefill_tokens_ptr,
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
num_reqs = tl.num_programs(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + req_idx)
|
||||
query_end = tl.load(query_start_loc_ptr + req_idx + 1)
|
||||
query_len = query_end - query_start
|
||||
seq_len = tl.load(seq_lens_ptr + req_idx)
|
||||
|
||||
# Get the true query length and next token after accounting for rejected tokens.
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
query_len -= num_rejected
|
||||
|
||||
num_sampled = tl.load(num_sampled_ptr + req_idx)
|
||||
if num_sampled > 0:
|
||||
next_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32)
|
||||
else:
|
||||
# Chunked prefilling.
|
||||
# Get the next prefill token.
|
||||
next_token = tl.load(next_prefill_tokens_ptr + req_state_idx)
|
||||
|
||||
# Shift target_input_ids by one.
|
||||
for i in range(1, query_len, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < query_len
|
||||
input_ids = tl.load(target_input_ids_ptr + query_start + block, mask=mask)
|
||||
tl.store(eagle_input_ids_ptr + query_start + block - 1, input_ids, mask=mask)
|
||||
|
||||
last_token_index = query_start + query_len - 1
|
||||
tl.store(last_token_indices_ptr + req_idx, last_token_index)
|
||||
tl.store(eagle_input_ids_ptr + last_token_index, next_token)
|
||||
|
||||
# Copy positions.
|
||||
for i in range(0, query_len, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < query_len
|
||||
target_pos = tl.load(target_positions_ptr + query_start + block, mask=mask)
|
||||
tl.store(eagle_positions_ptr + query_start + block, target_pos, mask=mask)
|
||||
|
||||
# Copy query start locations.
|
||||
tl.store(eagle_query_start_loc_ptr + req_idx, query_start)
|
||||
# Copy sequence lengths.
|
||||
tl.store(eagle_seq_lens_ptr + req_idx, seq_len)
|
||||
if req_idx == (num_reqs - 1):
|
||||
# Reset the current draft step to 0.
|
||||
tl.store(eagle_current_draft_step_ptr, 0)
|
||||
# Pad query_start_loc for CUDA graphs.
|
||||
for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs + 1
|
||||
tl.store(eagle_query_start_loc_ptr + block, query_end, mask=mask)
|
||||
# Pad seq_lens for CUDA graphs.
|
||||
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(eagle_seq_lens_ptr + block, 0, mask=mask)
|
||||
# Pad last_token_indices for CUDA graphs.
|
||||
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(last_token_indices_ptr + block, 0, mask=mask)
|
||||
|
||||
|
||||
def prepare_eagle_inputs(
|
||||
# [num_reqs]
|
||||
last_token_indices: torch.Tensor,
|
||||
current_draft_step: torch.Tensor,
|
||||
input_buffers: InputBuffers,
|
||||
input_batch: InputBatch,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
max_num_reqs,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
_prepare_eagle_inputs_kernel[(num_reqs,)](
|
||||
last_token_indices,
|
||||
current_draft_step,
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
input_batch.input_ids,
|
||||
input_batch.positions,
|
||||
input_batch.idx_mapping,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.seq_lens,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
return last_token_indices
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_eagle_decode_kernel(
|
||||
draft_tokens_ptr,
|
||||
draft_tokens_stride,
|
||||
target_seq_lens_ptr,
|
||||
num_rejected_ptr,
|
||||
input_ids_ptr,
|
||||
positions_ptr,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
max_model_len,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
num_reqs = tl.num_programs(0) - 1
|
||||
if req_idx == num_reqs:
|
||||
# Compute query_start_loc. Pad it with the last query_start_loc
|
||||
# for CUDA graphs.
|
||||
for i in range(0, max_num_reqs + 1, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
q = tl.where(block < num_reqs, block, num_reqs)
|
||||
mask = block < max_num_reqs + 1
|
||||
tl.store(query_start_loc_ptr + block, q, mask=mask)
|
||||
# Pad seq_lens for CUDA graphs.
|
||||
for i in range(req_idx, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(seq_lens_ptr + block, 0, mask=mask)
|
||||
return
|
||||
|
||||
# draft token -> input id.
|
||||
draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride)
|
||||
tl.store(input_ids_ptr + req_idx, draft_token)
|
||||
|
||||
# Compute position and seq_lens.
|
||||
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
|
||||
# if they reach the max model length.
|
||||
position = tl.load(positions_ptr + req_idx)
|
||||
position = tl.minimum(position + 1, max_model_len - 1)
|
||||
tl.store(positions_ptr + req_idx, position)
|
||||
|
||||
target_seq_len = tl.load(target_seq_lens_ptr + req_idx)
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
seq_len = target_seq_len - num_rejected
|
||||
seq_len = tl.minimum(seq_len + 1, max_model_len)
|
||||
tl.store(seq_lens_ptr + req_idx, seq_len)
|
||||
|
||||
|
||||
def prepare_eagle_decode(
|
||||
draft_tokens: torch.Tensor,
|
||||
target_seq_lens: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
input_buffers: InputBuffers,
|
||||
max_model_len: int,
|
||||
max_num_reqs: int,
|
||||
):
|
||||
num_reqs = draft_tokens.shape[0]
|
||||
_prepare_eagle_decode_kernel[(num_reqs + 1,)](
|
||||
draft_tokens,
|
||||
draft_tokens.stride(0),
|
||||
target_seq_lens,
|
||||
num_rejected,
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
max_model_len,
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _update_eagle_draft_inputs_kernel(
|
||||
output_draft_tokens_ptr,
|
||||
output_draft_tokens_stride,
|
||||
next_input_hidden_states_ptr,
|
||||
next_input_hidden_states_stride,
|
||||
input_ids_ptr,
|
||||
positions_ptr,
|
||||
seq_lens_ptr,
|
||||
draft_tokens_ptr,
|
||||
current_draft_step_ptr,
|
||||
hidden_states_ptr,
|
||||
hidden_states_stride,
|
||||
hidden_size,
|
||||
max_model_len,
|
||||
num_speculative_steps,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
|
||||
# Write the sampled draft token into self.draft_tokens[req_idx, step].
|
||||
draft_token = tl.load(draft_tokens_ptr + req_idx)
|
||||
step = tl.load(current_draft_step_ptr)
|
||||
tl.store(
|
||||
output_draft_tokens_ptr + req_idx * output_draft_tokens_stride + step,
|
||||
draft_token,
|
||||
)
|
||||
|
||||
if step >= num_speculative_steps - 1:
|
||||
# This is the final step. Skip updating draft forward inputs.
|
||||
return
|
||||
|
||||
# Write the sampled draft token into the input ids tensor for the next
|
||||
# forward pass.
|
||||
tl.store(input_ids_ptr + req_idx, draft_token)
|
||||
|
||||
# Copy hidden states into the input hidden states tensor for the next
|
||||
# forward pass.
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < hidden_size
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + req_idx * hidden_states_stride + block,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
next_input_hidden_states_ptr
|
||||
+ req_idx * next_input_hidden_states_stride
|
||||
+ block,
|
||||
hidden_states,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
# Increment position and seq_lens.
|
||||
# NOTE(woosuk): To prevent out-of-range access, we clamp these values
|
||||
# if they reach the max model length.
|
||||
position = tl.load(positions_ptr + req_idx)
|
||||
position = tl.minimum(position + 1, max_model_len - 1)
|
||||
tl.store(positions_ptr + req_idx, position)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + req_idx)
|
||||
seq_len = tl.minimum(seq_len + 1, max_model_len)
|
||||
tl.store(seq_lens_ptr + req_idx, seq_len)
|
||||
|
||||
|
||||
def update_eagle_draft_inputs(
|
||||
draft_tokens: torch.Tensor,
|
||||
current_draft_step: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
output_draft_tokens: torch.Tensor,
|
||||
next_input_hidden_states: torch.Tensor,
|
||||
input_buffers: InputBuffers,
|
||||
num_reqs: int,
|
||||
max_model_len: int,
|
||||
num_speculative_steps: int,
|
||||
):
|
||||
_, hidden_size = hidden_states.shape
|
||||
_update_eagle_draft_inputs_kernel[(num_reqs,)](
|
||||
output_draft_tokens,
|
||||
output_draft_tokens.stride(0),
|
||||
next_input_hidden_states,
|
||||
next_input_hidden_states.stride(0),
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.seq_lens,
|
||||
draft_tokens,
|
||||
current_draft_step,
|
||||
hidden_states,
|
||||
hidden_states.stride(0),
|
||||
hidden_size,
|
||||
max_model_len,
|
||||
num_speculative_steps,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> nn.Module:
|
||||
return load_eagle_model(target_model, self.vllm_config)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Gemma4 MTP (Multi-Token Prediction) speculator for speculative decoding.
|
||||
|
||||
The Gemma4 assistant model runs all decoder layers per draft step
|
||||
(producing one token), and all its attention layers share KV cache
|
||||
with the target model via cross-model KV sharing.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
# Gemma4 MTP is Q-only and reads K/V from the target's existing cache.
|
||||
# No new KV slots are written, so positions and seq_lens stay fixed.
|
||||
return False
|
||||
|
||||
@property
|
||||
def model_returns_tuple(self) -> bool:
|
||||
# forward() returns (draft_hidden_states, backbone_hidden_states).
|
||||
# The proposer uses draft_hidden_states for compute_logits and
|
||||
# backbone_hidden_states for the hidden-state feedback buffer.
|
||||
return True
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> nn.Module:
|
||||
draft_vllm_config = self._create_draft_vllm_config()
|
||||
with set_model_tag("eagle_head"):
|
||||
draft_model = get_model(
|
||||
vllm_config=draft_vllm_config,
|
||||
model_config=self.speculative_config.draft_model_config,
|
||||
load_config=self.speculative_config.draft_load_config,
|
||||
)
|
||||
self._setup_gemma4_kv_sharing(draft_model, target_attn_layer_names)
|
||||
self._share_embeddings(draft_model, target_model)
|
||||
return draft_model
|
||||
|
||||
def _create_draft_vllm_config(self) -> VllmConfig:
|
||||
"""Preserve the target's forced TRITON_ATTN backend for draft layers.
|
||||
|
||||
Gemma4 forces TRITON_ATTN due to heterogeneous head dimensions
|
||||
(head_dim=256 sliding, global_head_dim=512 full). The base class
|
||||
resets attention_config.backend to None for draft models, causing
|
||||
sliding layers to fall back to FLASH_ATTN which cannot handle
|
||||
KV-shared cache. Override to carry the target's backend through.
|
||||
"""
|
||||
draft_model_config = self.speculative_config.draft_model_config
|
||||
draft_vllm_config = replace(
|
||||
self.vllm_config,
|
||||
model_config=draft_model_config,
|
||||
)
|
||||
target_backend = self.vllm_config.attention_config.backend
|
||||
if target_backend is not None:
|
||||
draft_vllm_config = replace(
|
||||
draft_vllm_config,
|
||||
attention_config=replace(
|
||||
draft_vllm_config.attention_config,
|
||||
backend=target_backend,
|
||||
),
|
||||
)
|
||||
return draft_vllm_config
|
||||
|
||||
def _setup_gemma4_kv_sharing(
|
||||
self,
|
||||
model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> None:
|
||||
"""Wire draft layers to share KV with the target model.
|
||||
|
||||
Each draft decoder layer is mapped to the last non-KV-shared
|
||||
target layer of the same attention type (sliding or full).
|
||||
"""
|
||||
draft_config = self.speculative_config.draft_model_config.hf_config
|
||||
draft_text_config = draft_config.get_text_config()
|
||||
target_config = self.vllm_config.model_config.hf_config
|
||||
target_text_config = target_config.get_text_config()
|
||||
target_layer_types = getattr(target_text_config, "layer_types", [])
|
||||
|
||||
if not (hasattr(model, "model") and hasattr(model.model, "layers")):
|
||||
return
|
||||
|
||||
target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0)
|
||||
num_non_shared = len(target_layer_types) - target_num_kv_shared
|
||||
type_to_target_indices: dict[str, list[int]] = defaultdict(list)
|
||||
for idx, lt in enumerate(target_layer_types[:num_non_shared]):
|
||||
type_to_target_indices[lt].append(idx)
|
||||
|
||||
target_prefix = "model.layers"
|
||||
for name in target_attn_layer_names:
|
||||
if ".layers." in name:
|
||||
target_prefix = name.split(".layers.")[0] + ".layers"
|
||||
break
|
||||
|
||||
draft_layer_types = getattr(draft_text_config, "layer_types", [])
|
||||
for draft_idx, layer in enumerate(model.model.layers):
|
||||
if not hasattr(layer, "self_attn"):
|
||||
continue
|
||||
attn = getattr(layer.self_attn, "attn", None)
|
||||
if attn is None:
|
||||
continue
|
||||
|
||||
draft_layer_type = (
|
||||
draft_layer_types[draft_idx]
|
||||
if draft_idx < len(draft_layer_types)
|
||||
else "full_attention"
|
||||
)
|
||||
candidates = type_to_target_indices.get(draft_layer_type, [])
|
||||
if not candidates:
|
||||
logger.warning(
|
||||
"No target layer of type '%s' for draft layer %d",
|
||||
draft_layer_type,
|
||||
draft_idx,
|
||||
)
|
||||
continue
|
||||
|
||||
target_idx = candidates[-1]
|
||||
target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn"
|
||||
attn.kv_sharing_target_layer_name = target_layer_name
|
||||
logger.info(
|
||||
"Gemma4 MTP: draft layer %d (%s) -> %s",
|
||||
draft_idx,
|
||||
draft_layer_type,
|
||||
target_layer_name,
|
||||
)
|
||||
|
||||
def _share_embeddings(
|
||||
self,
|
||||
draft_model: nn.Module,
|
||||
target_model: nn.Module,
|
||||
) -> None:
|
||||
target_language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
if get_pp_group().world_size == 1:
|
||||
target_embed = getattr(target_language_model.model, "embed_tokens", None)
|
||||
if target_embed is not None:
|
||||
del draft_model.model.embed_tokens
|
||||
draft_model.model.embed_tokens = target_embed
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
|
||||
class MTPSpeculator(AutoRegressiveSpeculator):
|
||||
@property
|
||||
def model_returns_tuple(self) -> bool:
|
||||
return False
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> nn.Module:
|
||||
return load_eagle_model(target_model, self.vllm_config)
|
||||
@@ -0,0 +1,224 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
init_attn_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
|
||||
|
||||
class BaseSpeculator(ABC):
|
||||
@abstractmethod
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def capture(
|
||||
self,
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def propose(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
attn_metadata: dict[str, Any],
|
||||
slot_mappings: dict[str, torch.Tensor],
|
||||
# [num_tokens, hidden_size]
|
||||
last_hidden_states: torch.Tensor,
|
||||
# num_layers x [num_tokens, hidden_size]
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class DraftModelSpeculator(BaseSpeculator):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self.method = self.speculative_config.method
|
||||
self.num_speculative_steps = self.speculative_config.num_speculative_tokens
|
||||
self.draft_model_config = self.speculative_config.draft_model_config
|
||||
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.max_num_reqs = self.scheduler_config.max_num_seqs
|
||||
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.draft_max_seq_len = self.max_model_len
|
||||
# We need to get the hidden size from the draft model config because
|
||||
# the draft model's hidden size can be different from the target model's
|
||||
# hidden size (e.g., Llama 3.3 70B).
|
||||
self.hidden_size = self.draft_model_config.get_hidden_size()
|
||||
# Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP
|
||||
# draft the target's pre-hc_head (T, hc_mult * hidden_size) residual).
|
||||
# Non-HC models default to hc_mult=1 and are unaffected.
|
||||
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
|
||||
self.hidden_size = self.hidden_size * hc_mult
|
||||
self.vocab_size = self.draft_model_config.get_vocab_size()
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
|
||||
|
||||
# DP configuration
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
device=device,
|
||||
)
|
||||
self.idx_mapping = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self.temperature = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.float32, device=device
|
||||
)
|
||||
self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
self.draft_tokens = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
self.arange = torch.arange(
|
||||
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
self.draft_logits = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.vocab_size,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> nn.Module:
|
||||
pass
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
target_attn_layer_names = set(
|
||||
get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
).keys()
|
||||
)
|
||||
|
||||
self.model = self.load_draft_model(target_model, target_attn_layer_names)
|
||||
|
||||
all_attn_layers = set[str](
|
||||
get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
).keys()
|
||||
)
|
||||
self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
) -> None:
|
||||
self.model_state = model_state
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.attn_groups, _, _ = init_attn_backend(
|
||||
kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
active_layer_names=self.draft_attn_layer_names,
|
||||
)
|
||||
self.block_tables = block_tables
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
) -> dict[str, Any] | None:
|
||||
query_start_loc_cpu = torch.clamp(
|
||||
self.arange[: num_reqs_padded + 1], max=num_reqs
|
||||
)
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
]
|
||||
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded]
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=self.attn_groups,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_tokens=num_tokens_padded,
|
||||
query_start_loc_gpu=self.input_buffers.query_start_loc[
|
||||
: num_reqs_padded + 1
|
||||
],
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=1,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
max_seq_len=self.draft_max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def _copy_request_inputs(
|
||||
self,
|
||||
num_reqs: int,
|
||||
# [num_reqs]
|
||||
idx_mapping: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
) -> None:
|
||||
# Copy temperature, seeds, and idx mapping to the pre-allocated buffers.
|
||||
# NOTE(woosuk): For draft sampling, we only consider the temperature
|
||||
# and ignore the other sampling parameters such as top_k and top_p,
|
||||
# for simplicity and performance.
|
||||
# While this may slightly degrade the acceptance rate, it does not
|
||||
# affect the output distribution after rejection sampling.
|
||||
self.temperature.copy_(temperature)
|
||||
self.seeds.copy_(seeds)
|
||||
self.idx_mapping[:num_reqs].copy_(idx_mapping)
|
||||
Reference in New Issue
Block a user