diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 83e3072546..73e1cce56d 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -62,7 +62,6 @@ from vllm.v1.attention.backends.utils import ( KVCacheLayoutType, get_dcp_local_seq_lens, get_kv_cache_layout, - get_num_attention_heads_from_layers, get_per_layer_parameters, infer_global_hyperparameters, split_decodes_and_prefills, @@ -608,10 +607,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a" ) - # Compatible with models with non-uniform per-layer head counts. - self.num_qo_heads = get_num_attention_heads_from_layers( - vllm_config, layer_names - ) or self.model_config.get_num_attention_heads(self.vllm_config.parallel_config) + self.num_qo_heads = self.model_config.get_num_attention_heads( + self.vllm_config.parallel_config + ) self.num_kv_heads = self.kv_cache_spec.num_kv_heads self.head_dim = self.kv_cache_spec.head_size diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 008b74c9ff..716d56e817 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -30,10 +30,7 @@ from vllm.v1.attention.backend import ( CommonAttentionMetadata, MultipleOf, ) -from vllm.v1.attention.backends.utils import ( - get_kv_cache_layout, - get_num_attention_heads_from_layers, -) +from vllm.v1.attention.backends.utils import get_kv_cache_layout from vllm.v1.attention.ops.triton_prefill_attention import context_attention_fwd from vllm.v1.attention.ops.triton_reshape_and_cache_flash import ( triton_reshape_and_cache_flash, @@ -142,10 +139,9 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet self.block_size = kv_cache_spec.block_size model_config = vllm_config.model_config - # Compatible with models with non-uniform per-layer head counts. - self.num_heads_q = get_num_attention_heads_from_layers( - vllm_config, layer_names - ) or model_config.get_num_attention_heads(vllm_config.parallel_config) + self.num_heads_q = model_config.get_num_attention_heads( + vllm_config.parallel_config + ) self.num_heads_kv = model_config.get_num_kv_heads(vllm_config.parallel_config) self.headdim = model_config.get_head_size() diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index b73d17e8e5..d09c01eb90 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -136,32 +136,6 @@ def get_per_layer_parameters( return per_layer_params -def get_num_attention_heads_from_layers( - vllm_config: VllmConfig, layer_names: list[str] -) -> int | None: - """Per-TP-rank ``num_heads`` shared by the named Attention layers. - - Use in metadata builders whose plan-time allocations depend on the - head count: the model-wide ``get_num_attention_heads()`` is wrong - for models with non-uniform per-layer head counts. All layers in - one attention group must agree on ``num_heads``; this is asserted. - Returns ``None`` when no matching Attention layer is found. - """ - attn_layers = get_layers_from_vllm_config( - vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - layer_names, - ) - if not attn_layers: - return None - heads = {layer.impl.num_heads for layer in attn_layers.values()} - assert len(heads) == 1, ( - f"All layers in one attention group must share num_heads; " - f"got {heads} for {layer_names}." - ) - return heads.pop() - - def infer_global_hyperparameters( per_layer_params: dict[str, PerLayerParameters], ) -> PerLayerParameters: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index be0460c64a..367147b0b4 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -103,6 +103,7 @@ from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( set_eagle3_aux_hidden_state_layers, ) from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker @@ -307,7 +308,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.use_aux_hidden_state_outputs: assert self.speculative_config is not None set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config) - if self.speculator is not None: + if isinstance(self.speculator, DraftModelSpeculator): self.speculator.load_model(self.model) eplb_models_added = self.eplb.maybe_register_speculator( self.speculator, self.speculative_config, load_dummy_weights @@ -457,7 +458,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.speculator.init_cudagraph_manager(cudagraph_mode) check_attention_cp_compatibility(self.vllm_config) - if self.speculator is not None: + if isinstance(self.speculator, DraftModelSpeculator): # HACK(woosuk) self.speculator.set_attn( self.model_state, self.kv_cache_config, self.block_tables diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index 536b7526bd..bafb28c5cc 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -8,8 +8,21 @@ from vllm.config import VllmConfig def init_speculator(vllm_config: VllmConfig, device: torch.device): speculative_config = vllm_config.speculative_config assert speculative_config is not None - if speculative_config.use_eagle(): - from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator + if speculative_config.use_gemma4_mtp(): + from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import ( + Gemma4Speculator, + ) + + return Gemma4Speculator(vllm_config, device) + elif speculative_config.method == "mtp": + from vllm.v1.worker.gpu.spec_decode.mtp.speculator import MTPSpeculator + + return MTPSpeculator(vllm_config, device) + elif speculative_config.use_eagle(): + from vllm.v1.worker.gpu.spec_decode.eagle.speculator import ( + EagleSpeculator, + ) return EagleSpeculator(vllm_config, device) - raise NotImplementedError(f"{speculative_config.method} is not supported yet.") + else: + raise NotImplementedError(f"{speculative_config.method} is not supported yet.") diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/__init__.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py similarity index 92% rename from vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py rename to vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py index 2c400498be..15ab7430c9 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py @@ -19,8 +19,8 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.utils import AttentionGroup -class PrefillEagleCudaGraphManager(CudaGraphManager): - """Eagle CudaGraphManager for prefill, using pre-built attention states +class PrefillSpeculatorCudaGraphManager(CudaGraphManager): + """CudaGraphManager for draft prefill, using pre-built attention states from the target model's capture.""" def capture( @@ -56,9 +56,8 @@ class PrefillEagleCudaGraphManager(CudaGraphManager): super().capture(create_forward_fn, progress_bar_desc) -class DecodeEagleCudaGraphManager(CudaGraphManager): - """Eagle CudaGraphManager for decode draft generation, building its own - attention metadata from scratch.""" +class DecodeSpeculatorCudaGraphManager(CudaGraphManager): + """CudaGraphManager for draft decode, building its own attention metadata.""" def capture( self, diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py new file mode 100644 index 0000000000..868540437b --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -0,0 +1,795 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +import torch + +from vllm.config import VllmConfig +from vllm.config.compilation import CUDAGraphMode +from vllm.forward_context import BatchDescriptor, set_forward_context +from vllm.logger import init_logger +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.triton_utils import tl, triton +from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer +from vllm.v1.worker.gpu.cudagraph_utils import ( + AttentionStatePair, + BatchExecutionDescriptor, + get_uniform_token_count, +) +from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample +from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( + DecodeSpeculatorCudaGraphManager, + PrefillSpeculatorCudaGraphManager, +) +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator + +logger = init_logger(__name__) + + +class AutoRegressiveSpeculator(DraftModelSpeculator): + def __init__(self, vllm_config: VllmConfig, device: torch.device): + super().__init__(vllm_config, device) + + self.hidden_states = torch.zeros( + self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device + ) + self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device) + self.last_token_indices = torch.zeros( + self.max_num_reqs, dtype=torch.int64, device=device + ) + + self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( + self.draft_model_config + ) + if self.supports_mm_inputs: + self.inputs_embeds = torch.zeros( + self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device + ) + + self.prefill_cudagraph_manager: PrefillSpeculatorCudaGraphManager | None = None + self.decode_cudagraph_manager: DecodeSpeculatorCudaGraphManager | None = None + + @property + def advance_draft_positions(self) -> bool: + """ + Whether to increment positions and seq_lens between draft steps. + + True for Eagle/standard MTP (each step produces new KV). + False for Gemma4 MTP (Q-only, shares target KV, constant positions). + """ + return True + + @property + def model_returns_tuple(self) -> bool: + """ + Whether the draft model's forward() returns a tuple. + + True: returns (last_hidden_states, hidden_states) — Eagle, Gemma4 MTP. + False: returns a single tensor used for both — standard MTP (DeepSeek). + """ + return True + + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: + # Initialize cudagraph manager for draft prefill (draft position 0). + self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager( + self.vllm_config, + self.device, + cudagraph_mode, + self.num_speculative_steps + 1, + ) + + # PIECEWISE cudagraphs are not supported for draft decodes. + if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL: + cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY + else: + cudagraph_mode = CUDAGraphMode.NONE + + # Initialize cudagraph manager for draft decodes (draft positions > 0). + self.decode_cudagraph_manager = DecodeSpeculatorCudaGraphManager( + self.vllm_config, + self.device, + cudagraph_mode, + decode_query_len=1, + ) + + def capture( + self, + attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], + ) -> None: + logger.info("Capturing model for speculator...") + # Reset indices to zeros to prevent stale values from prior + # dummy runs to cause out-of-bounds indexing during capture. + self.last_token_indices.zero_() + + # Capture the prefill routine (model forward + compute_logits + + # sample). + # For FULL graphs, the entire routine is recorded as one graph. + # For PIECEWISE, only the model's compiled regions are captured + # and the rest (compute_logits, gumbel_sample) runs eagerly. + assert self.prefill_cudagraph_manager is not None + if self.prefill_cudagraph_manager.use_breakable_cg: + self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model) + self.prefill_cudagraph_manager.capture( + self._prefill, + attn_states, + progress_bar_desc="Capturing prefill CUDA graphs", + ) + + if self.num_speculative_steps == 1: + return + + # Capture the decode draft generation routine (model forward + + # sample + update_draft_inputs) for a single + # step. + assert self.decode_cudagraph_manager is not None + self.decode_cudagraph_manager.capture( + self._generate_draft, + self.model_state, + self.input_buffers, + self.block_tables, + self.attn_groups, + self.kv_cache_config, + progress_bar_desc="Capturing decode CUDA graphs", + ) + + @torch.inference_mode() + def propose( + self, + input_batch: InputBatch, + attn_metadata: dict[str, Any], + slot_mappings: dict[str, torch.Tensor], + # [num_tokens, hidden_size] + last_hidden_states: torch.Tensor, + # num_layers x [num_tokens, hidden_size] + aux_hidden_states: list[torch.Tensor] | None, + # [num_reqs] + num_sampled: torch.Tensor, + # [num_reqs] + num_rejected: torch.Tensor, + # [max_num_reqs] + last_sampled: torch.Tensor, + # [max_num_reqs] + next_prefill_tokens: torch.Tensor, + # [max_num_reqs] + temperature: torch.Tensor, + # [max_num_reqs] + seeds: torch.Tensor, + num_tokens_across_dp: torch.Tensor | None = None, + dummy_run: bool = False, + skip_attn_for_dummy_run: bool = False, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + is_profile: bool = False, + ) -> torch.Tensor: + num_tokens = input_batch.num_tokens_after_padding + num_reqs = input_batch.num_reqs + max_query_len = input_batch.num_scheduled_tokens.max() + max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item() + self.draft_max_seq_len = min( + max_seq_len + self.num_speculative_steps, self.max_model_len + ) + + # NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the + # number of rejected tokens, we maintain the size of input_ids and + # hidden_states the same as the target model's. This means, we pad each + # request's query length to include any rejected positions. By doing so, + # we can also reuse the attention metadata (e.g., query_start_loc, + # seq_lens) of the target model. + if aux_hidden_states: + assert self.method == "eagle3" + hidden_states = self.model.combine_hidden_states( + torch.cat(aux_hidden_states, dim=-1) + ) + else: + hidden_states = last_hidden_states + self.hidden_states[:num_tokens].copy_(hidden_states) + + self._copy_request_inputs( + num_reqs, + input_batch.idx_mapping, + temperature, + seeds, + ) + + # Get the input ids and last token indices for the speculator. + prepare_prefill_inputs( + self.last_token_indices, + self.current_draft_step, + self.input_buffers, + input_batch, + num_sampled, + num_rejected, + last_sampled, + next_prefill_tokens, + self.max_num_reqs, + ) + + # When all requests are decoding (no true prefills), each has + # num_speculative_steps + 1 tokens, enabling FULL graph replay. + uniform_token_count = get_uniform_token_count( + num_reqs, + # Use the actual number of tokens without padding added by + # the target model during FULL cudagraph. + input_batch.num_tokens, + max_query_len, + ) + prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( + self.prefill_cudagraph_manager, + num_reqs, + num_tokens, + uniform_token_count, + dp_size=self.dp_size, + dp_rank=self.dp_rank, + need_eager=is_profile, + ) + + if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL: + # Replay the full graph for draft prefill. + assert self.prefill_cudagraph_manager is not None + self.prefill_cudagraph_manager.run_fullgraph(prefill_batch_desc) + else: + # The target model's attention metadata and slot mappings + # can directly be used for draft prefill, because of the + # identical batch shape and KV cache layout. + self._prefill( + num_reqs, + prefill_batch_desc.num_tokens, + attn_metadata, + slot_mappings, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=prefill_batch_desc.cg_mode, + mm_inputs=mm_inputs, + ) + + if self.num_speculative_steps == 1: + # Early exit. + return self.draft_tokens[:num_reqs, :1] + + # Prepare the inputs for the decode steps. + prepare_decode_inputs( + self.draft_tokens[:num_reqs, 0], + input_batch.seq_lens, + num_rejected, + self.input_buffers, + self.max_model_len, + self.max_num_reqs, + advance_draft_positions=self.advance_draft_positions, + ) + + # Each request produces exactly 1 token per draft generation step, + # enabling FULL graph replay. + decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( + self.decode_cudagraph_manager, + num_reqs, + num_reqs, + uniform_token_count=1, + dp_size=self.dp_size, + dp_rank=self.dp_rank, + need_eager=is_profile, + ) + + # Generate the remaining num_speculative_steps - 1 draft tokens. + self._multi_step_decode( + num_reqs, + dummy_run and skip_attn_for_dummy_run, + decode_batch_desc, + num_tokens_across_dp, + ) + + return self.draft_tokens[:num_reqs] + + def sample_draft( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + idx_mapping: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, + draft_step: torch.Tensor, + draft_logits: torch.Tensor | None, + ) -> torch.Tensor: + logits = self.model.compute_logits(hidden_states) + if draft_logits is not None: + # NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise + # used for draft and target sampling. + return gumbel_sample( + logits, + idx_mapping, + temperature, + seeds, + positions + 1, + apply_temperature=True, + output_processed_logits=draft_logits, + output_processed_logits_col=draft_step, + use_fp64=self.use_fp64_gumbel, + ) + else: + return logits.argmax(dim=-1) + + @torch.inference_mode() + def _run_model( + self, + num_tokens: int, + attn_metadata: dict[str, Any] | None, + slot_mappings: dict[str, torch.Tensor] | None, + num_tokens_across_dp: torch.Tensor | None, + cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + batch_descriptor = BatchDescriptor(num_tokens=num_tokens) + with set_forward_context( + attn_metadata, + self.vllm_config, + num_tokens=num_tokens, + cudagraph_runtime_mode=cudagraph_runtime_mode, + num_tokens_across_dp=num_tokens_across_dp, + slot_mapping=slot_mappings, + batch_descriptor=batch_descriptor, + ): + inputs_embeds = None + if self.supports_mm_inputs: + # Merge multimodal embeddings with input ids. + mm_embeds, is_mm_embed = mm_inputs or (None, None) + num_input_tokens = ( + is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens + ) + self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids( + self.input_buffers.input_ids[:num_input_tokens], + multimodal_embeddings=mm_embeds, + is_multimodal=is_mm_embed, + ) + inputs_embeds = self.inputs_embeds[:num_tokens] + + model_inputs = dict( + input_ids=self.input_buffers.input_ids[:num_tokens], + positions=self.input_buffers.positions[:num_tokens], + hidden_states=self.hidden_states[:num_tokens], + inputs_embeds=inputs_embeds, + ) + if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE: + # Draft prefill with PIECEWISE cudagraph (compiled PW or breakable), + # chosen inside run_pw_graph. + assert self.prefill_cudagraph_manager is not None + ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph( + self.model, model_inputs + ) + else: + # Eager (NONE): call the raw model directly. + ret_hidden_states = self.model(**model_inputs) + if self.model_returns_tuple: + last_hidden_states, hidden_states = ret_hidden_states + else: + last_hidden_states = ret_hidden_states + hidden_states = ret_hidden_states + return last_hidden_states, hidden_states + + def _prefill( + self, + num_reqs: int, + num_tokens: int, + attn_metadata: dict[str, Any] | None, + slot_mappings: dict[str, torch.Tensor] | None, + num_tokens_across_dp: torch.Tensor | None, + cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + ) -> None: + last_token_indices = self.last_token_indices[:num_reqs] + positions = self.input_buffers.positions[last_token_indices] + idx_mapping = self.idx_mapping[:num_reqs] + + last_hidden_states, hidden_states = self._run_model( + num_tokens, + attn_metadata, + slot_mappings, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=cudagraph_runtime_mode, + mm_inputs=mm_inputs, + ) + sample_hidden_states = last_hidden_states[last_token_indices] + + self.draft_tokens[:num_reqs, 0] = self.sample_draft( + sample_hidden_states, + positions, + idx_mapping, + self.temperature, + self.seeds, + self.current_draft_step, + self.draft_logits, + ) + self.hidden_states[:num_reqs] = hidden_states[last_token_indices] + self.input_buffers.positions[:num_reqs] = positions + + def _multi_step_decode( + self, + num_reqs: int, + skip_attn: bool, + batch_desc: BatchExecutionDescriptor, + num_tokens_across_dp: torch.Tensor | None, + ) -> None: + positions = self.input_buffers.positions[:num_reqs] + query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1] + idx_mapping = self.idx_mapping[:num_reqs] + + attn_metadata = None + slot_mappings_by_layer = None + for step in range(1, self.num_speculative_steps): + # Rebuild every step when positions advance, or just once + # on the first step when positions are constant (Gemma4 MTP). + if not skip_attn and (self.advance_draft_positions or step == 1): + slot_mappings = self.block_tables.compute_slot_mappings( + idx_mapping, + query_start_loc, + positions, + batch_desc.num_tokens, + ) + slot_mappings_by_layer = build_slot_mappings_by_layer( + slot_mappings, self.kv_cache_config + ) + attn_metadata = self._build_draft_attn_metadata( + num_reqs=num_reqs, + num_reqs_padded=batch_desc.num_reqs or num_reqs, + num_tokens_padded=batch_desc.num_tokens, + ) + + # Update the current draft step. + self.current_draft_step.fill_(step) + + # Generate draft tokens for the current step. + if batch_desc.cg_mode == CUDAGraphMode.FULL: + assert self.decode_cudagraph_manager is not None + self.decode_cudagraph_manager.run_fullgraph(batch_desc) + else: + self._generate_draft( + num_reqs, + batch_desc.num_tokens, + attn_metadata, + slot_mappings_by_layer, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=batch_desc.cg_mode, + ) + + def _generate_draft( + self, + num_reqs: int, + num_tokens_padded: int, + attn_metadata: dict[str, Any] | None, + slot_mappings: dict[str, torch.Tensor] | None, + num_tokens_across_dp: torch.Tensor | None, + cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, + ) -> None: + idx_mapping = self.idx_mapping[:num_reqs] + positions = self.input_buffers.positions[:num_reqs] + # Run the draft model forward pass. + last_hidden_states, hidden_states = self._run_model( + num_tokens_padded, + attn_metadata, + slot_mappings, + num_tokens_across_dp, + cudagraph_runtime_mode, + ) + last_hidden_states = last_hidden_states[:num_reqs] + + # Sample the draft tokens. + draft_tokens = self.sample_draft( + last_hidden_states, + positions, + idx_mapping, + self.temperature, + self.seeds, + self.current_draft_step, + self.draft_logits, + ) + + # Update the inputs for the next step. + update_draft_inputs( + draft_tokens, + self.current_draft_step, + hidden_states, + self.draft_tokens, + self.hidden_states, + self.input_buffers, + num_reqs, + self.max_model_len, + self.num_speculative_steps, + advance_draft_positions=self.advance_draft_positions, + ) + + +@triton.jit +def _prepare_prefill_inputs_kernel( + last_token_indices_ptr, + draft_current_step_ptr, + draft_input_ids_ptr, + draft_positions_ptr, + draft_query_start_loc_ptr, + draft_seq_lens_ptr, + target_input_ids_ptr, + target_positions_ptr, + idx_mapping_ptr, + last_sampled_ptr, + next_prefill_tokens_ptr, + num_sampled_ptr, + num_rejected_ptr, + query_start_loc_ptr, + seq_lens_ptr, + max_num_reqs, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + num_reqs = tl.num_programs(0) + req_state_idx = tl.load(idx_mapping_ptr + req_idx) + + query_start = tl.load(query_start_loc_ptr + req_idx) + query_end = tl.load(query_start_loc_ptr + req_idx + 1) + query_len = query_end - query_start + seq_len = tl.load(seq_lens_ptr + req_idx) + + # Get the true query length and next token after accounting for rejected tokens. + num_rejected = tl.load(num_rejected_ptr + req_idx) + query_len -= num_rejected + + num_sampled = tl.load(num_sampled_ptr + req_idx) + if num_sampled > 0: + next_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32) + else: + # Chunked prefilling. + # Get the next prefill token. + next_token = tl.load(next_prefill_tokens_ptr + req_state_idx) + + # Shift target_input_ids by one. + for i in range(1, query_len, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < query_len + input_ids = tl.load(target_input_ids_ptr + query_start + block, mask=mask) + tl.store(draft_input_ids_ptr + query_start + block - 1, input_ids, mask=mask) + + last_token_index = query_start + query_len - 1 + tl.store(last_token_indices_ptr + req_idx, last_token_index) + tl.store(draft_input_ids_ptr + last_token_index, next_token) + + # Copy positions. + for i in range(0, query_len, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < query_len + target_pos = tl.load(target_positions_ptr + query_start + block, mask=mask) + tl.store(draft_positions_ptr + query_start + block, target_pos, mask=mask) + + # Copy query start locations. + tl.store(draft_query_start_loc_ptr + req_idx, query_start) + # Copy sequence lengths. + tl.store(draft_seq_lens_ptr + req_idx, seq_len) + if req_idx == (num_reqs - 1): + # Reset the current draft step to 0. + tl.store(draft_current_step_ptr, 0) + # Pad query_start_loc for CUDA graphs. + for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + 1 + tl.store(draft_query_start_loc_ptr + block, query_end, mask=mask) + # Pad seq_lens for CUDA graphs. + for i in range(num_reqs, max_num_reqs, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + tl.store(draft_seq_lens_ptr + block, 0, mask=mask) + # Pad last_token_indices for CUDA graphs. + for i in range(num_reqs, max_num_reqs, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + tl.store(last_token_indices_ptr + block, 0, mask=mask) + + +def prepare_prefill_inputs( + # [num_reqs] + last_token_indices: torch.Tensor, + current_draft_step: torch.Tensor, + input_buffers: InputBuffers, + input_batch: InputBatch, + # [num_reqs] + num_sampled: torch.Tensor, + # [num_reqs] + num_rejected: torch.Tensor, + # [max_num_reqs] + last_sampled: torch.Tensor, + # [max_num_reqs] + next_prefill_tokens: torch.Tensor, + max_num_reqs, +) -> torch.Tensor: + num_reqs = input_batch.num_reqs + _prepare_prefill_inputs_kernel[(num_reqs,)]( + last_token_indices, + current_draft_step, + input_buffers.input_ids, + input_buffers.positions, + input_buffers.query_start_loc, + input_buffers.seq_lens, + input_batch.input_ids, + input_batch.positions, + input_batch.idx_mapping, + last_sampled, + next_prefill_tokens, + num_sampled, + num_rejected, + input_batch.query_start_loc, + input_batch.seq_lens, + max_num_reqs, + BLOCK_SIZE=1024, + ) + return last_token_indices + + +@triton.jit +def _prepare_decode_inputs_kernel( + draft_tokens_ptr, + draft_tokens_stride, + target_seq_lens_ptr, + num_rejected_ptr, + input_ids_ptr, + positions_ptr, + query_start_loc_ptr, + seq_lens_ptr, + max_model_len, + max_num_reqs, + BLOCK_SIZE: tl.constexpr, + ADVANCE_DRAFT_POSITIONS: tl.constexpr, +): + req_idx = tl.program_id(0) + num_reqs = tl.num_programs(0) - 1 + if req_idx == num_reqs: + # Compute query_start_loc. Pad it with the last query_start_loc + # for CUDA graphs. + for i in range(0, max_num_reqs + 1, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + q = tl.where(block < num_reqs, block, num_reqs) + mask = block < max_num_reqs + 1 + tl.store(query_start_loc_ptr + block, q, mask=mask) + # Pad seq_lens for CUDA graphs. + for i in range(req_idx, max_num_reqs, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + tl.store(seq_lens_ptr + block, 0, mask=mask) + return + + # draft token -> input id. + draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride) + tl.store(input_ids_ptr + req_idx, draft_token) + + if ADVANCE_DRAFT_POSITIONS: + # Compute position and seq_lens. + # NOTE(woosuk): To prevent out-of-range access, we clamp these values + # if they reach the max model length. + position = tl.load(positions_ptr + req_idx) + position = tl.minimum(position + 1, max_model_len - 1) + tl.store(positions_ptr + req_idx, position) + + target_seq_len = tl.load(target_seq_lens_ptr + req_idx) + num_rejected = tl.load(num_rejected_ptr + req_idx) + seq_len = target_seq_len - num_rejected + seq_len = tl.minimum(seq_len + 1, max_model_len) + tl.store(seq_lens_ptr + req_idx, seq_len) + + +def prepare_decode_inputs( + draft_tokens: torch.Tensor, + target_seq_lens: torch.Tensor, + num_rejected: torch.Tensor, + input_buffers: InputBuffers, + max_model_len: int, + max_num_reqs: int, + advance_draft_positions: bool = True, +): + num_reqs = draft_tokens.shape[0] + _prepare_decode_inputs_kernel[(num_reqs + 1,)]( + draft_tokens, + draft_tokens.stride(0), + target_seq_lens, + num_rejected, + input_buffers.input_ids, + input_buffers.positions, + input_buffers.query_start_loc, + input_buffers.seq_lens, + max_model_len, + max_num_reqs, + BLOCK_SIZE=1024, + ADVANCE_DRAFT_POSITIONS=advance_draft_positions, + ) + + +@triton.jit +def _update_draft_inputs_kernel( + output_draft_tokens_ptr, + output_draft_tokens_stride, + next_input_hidden_states_ptr, + next_input_hidden_states_stride, + input_ids_ptr, + positions_ptr, + seq_lens_ptr, + draft_tokens_ptr, + current_draft_step_ptr, + hidden_states_ptr, + hidden_states_stride, + hidden_size, + max_model_len, + num_speculative_steps, + BLOCK_SIZE: tl.constexpr, + ADVANCE_DRAFT_POSITIONS: tl.constexpr, +): + req_idx = tl.program_id(0) + + # Write the sampled draft token into self.draft_tokens[req_idx, step]. + draft_token = tl.load(draft_tokens_ptr + req_idx) + step = tl.load(current_draft_step_ptr) + tl.store( + output_draft_tokens_ptr + req_idx * output_draft_tokens_stride + step, + draft_token, + ) + + if step >= num_speculative_steps - 1: + # This is the final step. Skip updating draft forward inputs. + return + + # Write the sampled draft token into the input ids tensor for the next + # forward pass. + tl.store(input_ids_ptr + req_idx, draft_token) + + # Copy hidden states into the input hidden states tensor for the next + # forward pass. + for i in range(0, hidden_size, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < hidden_size + hidden_states = tl.load( + hidden_states_ptr + req_idx * hidden_states_stride + block, + mask=mask, + ) + tl.store( + next_input_hidden_states_ptr + + req_idx * next_input_hidden_states_stride + + block, + hidden_states, + mask=mask, + ) + + if ADVANCE_DRAFT_POSITIONS: + # Increment position and seq_lens. + # NOTE(woosuk): To prevent out-of-range access, we clamp these values + # if they reach the max model length. + position = tl.load(positions_ptr + req_idx) + position = tl.minimum(position + 1, max_model_len - 1) + tl.store(positions_ptr + req_idx, position) + + seq_len = tl.load(seq_lens_ptr + req_idx) + seq_len = tl.minimum(seq_len + 1, max_model_len) + tl.store(seq_lens_ptr + req_idx, seq_len) + + +def update_draft_inputs( + draft_tokens: torch.Tensor, + current_draft_step: torch.Tensor, + hidden_states: torch.Tensor, + output_draft_tokens: torch.Tensor, + next_input_hidden_states: torch.Tensor, + input_buffers: InputBuffers, + num_reqs: int, + max_model_len: int, + num_speculative_steps: int, + advance_draft_positions: bool = True, +): + _, hidden_size = hidden_states.shape + _update_draft_inputs_kernel[(num_reqs,)]( + output_draft_tokens, + output_draft_tokens.stride(0), + next_input_hidden_states, + next_input_hidden_states.stride(0), + input_buffers.input_ids, + input_buffers.positions, + input_buffers.seq_lens, + draft_tokens, + current_draft_step, + hidden_states, + hidden_states.stride(0), + hidden_size, + max_model_len, + num_speculative_steps, + BLOCK_SIZE=1024, + ADVANCE_DRAFT_POSITIONS=advance_draft_positions, + ) diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index 3f88c89d72..e878872e62 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -1,903 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any -import torch import torch.nn as nn -from vllm.config import VllmConfig, get_layers_from_vllm_config -from vllm.config.compilation import CUDAGraphMode -from vllm.forward_context import BatchDescriptor, set_forward_context -from vllm.logger import init_logger -from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.triton_utils import tl, triton -from vllm.v1.kv_cache_interface import KVCacheConfig -from vllm.v1.worker.gpu.attn_utils import ( - build_attn_metadata, - build_slot_mappings_by_layer, - init_attn_backend, -) -from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.cudagraph_utils import ( - AttentionStatePair, - BatchExecutionDescriptor, - get_uniform_token_count, -) -from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp -from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers -from vllm.v1.worker.gpu.model_states.interface import ModelState -from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample -from vllm.v1.worker.gpu.spec_decode.eagle.cudagraph import ( - DecodeEagleCudaGraphManager, - PrefillEagleCudaGraphManager, +from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( + AutoRegressiveSpeculator, ) from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model -logger = init_logger(__name__) - -class EagleSpeculator: - def __init__(self, vllm_config: VllmConfig, device: torch.device): - self.vllm_config = vllm_config - self.device = device - - self.speculative_config = vllm_config.speculative_config - assert self.speculative_config is not None - self.method = self.speculative_config.method - self.num_speculative_steps = self.speculative_config.num_speculative_tokens - self.draft_model_config = self.speculative_config.draft_model_config - - self.scheduler_config = vllm_config.scheduler_config - self.max_num_reqs = self.scheduler_config.max_num_seqs - self.max_num_tokens = self.scheduler_config.max_num_batched_tokens - self.max_model_len = vllm_config.model_config.max_model_len - self.draft_max_seq_len = self.max_model_len - # We need to get the hidden size from the draft model config because - # the draft model's hidden size can be different from the target model's - # hidden size (e.g., Llama 3.3 70B). - self.hidden_size = self.draft_model_config.get_hidden_size() - # Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP - # draft the target's pre-hc_head (T, hc_mult * hidden_size) residual). - # Non-HC models default to hc_mult=1 and are unaffected. - hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1) - self.hidden_size = self.hidden_size * hc_mult - self.vocab_size = self.draft_model_config.get_vocab_size() - self.dtype = vllm_config.model_config.dtype - self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel - - # DP configuration - self.dp_size = vllm_config.parallel_config.data_parallel_size - self.dp_rank = vllm_config.parallel_config.data_parallel_rank - - self.input_buffers = InputBuffers( - max_num_reqs=self.max_num_reqs, - max_num_tokens=self.max_num_tokens, - device=device, - ) - self.hidden_states = torch.zeros( - self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device - ) - self.idx_mapping = torch.zeros( - self.max_num_reqs, dtype=torch.int32, device=device - ) - self.temperature = torch.zeros( - self.max_num_reqs, dtype=torch.float32, device=device - ) - self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device) - self.draft_tokens = torch.zeros( - self.max_num_reqs, - self.num_speculative_steps, - dtype=torch.int64, - device=device, - ) - self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device) - self.last_token_indices = torch.zeros( - self.max_num_reqs, dtype=torch.int64, device=device - ) - self.arange = torch.arange( - self.max_num_reqs + 1, dtype=torch.int32, device="cpu" - ) - - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - self.draft_model_config - ) - if self.supports_mm_inputs: - self.inputs_embeds = torch.zeros( - self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device - ) - - self.draft_logits: torch.Tensor | None = None - if self.speculative_config.draft_sample_method == "probabilistic": - self.draft_logits = torch.zeros( - self.max_num_reqs, - self.num_speculative_steps, - self.vocab_size, - dtype=torch.float32, - device=device, - ) - - self.prefill_cudagraph_manager: PrefillEagleCudaGraphManager | None = None - self.decode_cudagraph_manager: DecodeEagleCudaGraphManager | None = None - - def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: - cudagraph_mode = self.vllm_config.compilation_config.cudagraph_mode - # Initialize cudagraph manager for draft prefill (draft position 0). - self.prefill_cudagraph_manager = PrefillEagleCudaGraphManager( - self.vllm_config, - self.device, - cudagraph_mode, - self.num_speculative_steps + 1, - ) - - # PIECEWISE cudagraphs are not supported for eagle draft decodes. - # PIECEWISE pads num_tokens to the next capture size without padding - # num_reqs, which can cause attention backends to read past the - # valid per-request metadata (e.g. FlashInfer's kv_indptr buffer). - if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL: - cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY - else: - cudagraph_mode = CUDAGraphMode.NONE - - # Initialize cudagraph manager for draft decodes (draft positions > 0). - self.decode_cudagraph_manager = DecodeEagleCudaGraphManager( - self.vllm_config, - self.device, - cudagraph_mode, - decode_query_len=1, - ) - - def load_model(self, target_model: nn.Module) -> None: - target_attn_layer_names = get_layers_from_vllm_config( - self.vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - ).keys() - - self.model = load_eagle_model(target_model, self.vllm_config) - - all_attn_layers = get_layers_from_vllm_config( - self.vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - ).keys() - self.draft_attn_layer_names = set(all_attn_layers) - set( - target_attn_layer_names - ) - - def set_attn( +class EagleSpeculator(AutoRegressiveSpeculator): + def load_draft_model( self, - model_state: ModelState, - kv_cache_config: KVCacheConfig, - block_tables: BlockTables, - ) -> None: - self.model_state = model_state - self.kv_cache_config = kv_cache_config - self.attn_groups, _, _ = init_attn_backend( - kv_cache_config, - self.vllm_config, - self.device, - active_layer_names=self.draft_attn_layer_names, - ) - self.block_tables = block_tables - - @torch.inference_mode() - def run_model( - self, - num_tokens: int, - attn_metadata: dict[str, Any] | None, - slot_mappings: dict[str, torch.Tensor] | None, - num_tokens_across_dp: torch.Tensor | None, - cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, - mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - batch_descriptor = BatchDescriptor(num_tokens=num_tokens) - with set_forward_context( - attn_metadata, - self.vllm_config, - num_tokens=num_tokens, - cudagraph_runtime_mode=cudagraph_runtime_mode, - num_tokens_across_dp=num_tokens_across_dp, - slot_mapping=slot_mappings, - batch_descriptor=batch_descriptor, - ): - inputs_embeds = None - if self.supports_mm_inputs: - # Merge multimodal embeddings with input ids. - mm_embeds, is_mm_embed = mm_inputs or (None, None) - num_input_tokens = ( - is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens - ) - self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids( - self.input_buffers.input_ids[:num_input_tokens], - multimodal_embeddings=mm_embeds, - is_multimodal=is_mm_embed, - ) - inputs_embeds = self.inputs_embeds[:num_tokens] - - model_inputs = dict( - input_ids=self.input_buffers.input_ids[:num_tokens], - positions=self.input_buffers.positions[:num_tokens], - hidden_states=self.hidden_states[:num_tokens], - inputs_embeds=inputs_embeds, - ) - if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE: - # Draft prefill with PIECEWISE cudagraph (compiled PW or breakable), - # chosen inside run_pw_graph. - assert self.prefill_cudagraph_manager is not None - ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph( - self.model, model_inputs - ) - else: - # Eager (NONE): call the raw model directly. - ret_hidden_states = self.model(**model_inputs) - if self.method == "mtp": - last_hidden_states = ret_hidden_states - hidden_states = ret_hidden_states - else: - last_hidden_states, hidden_states = ret_hidden_states - return last_hidden_states, hidden_states - - def _sample_draft( - self, - logits: torch.Tensor, - idx_mapping: torch.Tensor, - pos: torch.Tensor, - draft_step: torch.Tensor, - draft_logits: torch.Tensor | None, - ) -> torch.Tensor: - if draft_logits is not None: - # NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise - # used for draft and target sampling. - return gumbel_sample( - logits, - idx_mapping, - self.temperature, - self.seeds, - pos + 1, - apply_temperature=True, - output_processed_logits=draft_logits, - output_processed_logits_col=draft_step, - use_fp64=self.use_fp64_gumbel, - ) - else: - return logits.argmax(dim=-1) - - def prefill( - self, - num_reqs: int, - num_tokens: int, - attn_metadata: dict[str, Any] | None, - slot_mappings: dict[str, torch.Tensor] | None, - num_tokens_across_dp: torch.Tensor | None, - cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, - mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, - ) -> None: - last_token_indices = self.last_token_indices[:num_reqs] - pos = self.input_buffers.positions[last_token_indices] - idx_mapping = self.idx_mapping[:num_reqs] - - last_hidden_states, hidden_states = self.run_model( - num_tokens, - attn_metadata, - slot_mappings, - num_tokens_across_dp=num_tokens_across_dp, - cudagraph_runtime_mode=cudagraph_runtime_mode, - mm_inputs=mm_inputs, - ) - sample_hidden_states = last_hidden_states[last_token_indices] - logits = self.model.compute_logits(sample_hidden_states) - - self.draft_tokens[:num_reqs, 0] = self._sample_draft( - logits, - idx_mapping, - pos, - self.current_draft_step, - self.draft_logits, - ) - self.hidden_states[:num_reqs] = hidden_states[last_token_indices] - self.input_buffers.positions[:num_reqs] = pos - - def multi_step_decode( - self, - num_reqs: int, - skip_attn: bool, - batch_desc: BatchExecutionDescriptor, - num_tokens_across_dp: torch.Tensor | None, - ) -> None: - positions = self.input_buffers.positions[:num_reqs] - query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1] - idx_mapping = self.idx_mapping[:num_reqs] - - for step in range(1, self.num_speculative_steps): - attn_metadata = None - slot_mappings_by_layer = None - if not skip_attn: - # Build attention metadata and slot mappings for each draft - # decode step. It is necessary to rebuild the attention - # metadata even when replaying the FULL graph so that any - # attention metadata builder state is updated. - slot_mappings = self.block_tables.compute_slot_mappings( - idx_mapping, - query_start_loc, - positions, - batch_desc.num_tokens, - ) - slot_mappings_by_layer = build_slot_mappings_by_layer( - slot_mappings, self.kv_cache_config - ) - attn_metadata = self._build_draft_attn_metadata( - num_reqs=num_reqs, - num_reqs_padded=batch_desc.num_reqs or num_reqs, - num_tokens_padded=batch_desc.num_tokens, - ) - - # Update the current draft step. - self.current_draft_step.fill_(step) - - # Generate draft tokens for the current step. - if batch_desc.cg_mode == CUDAGraphMode.FULL: - assert self.decode_cudagraph_manager is not None - self.decode_cudagraph_manager.run_fullgraph(batch_desc) - else: - self.generate_draft( - num_reqs, - batch_desc.num_tokens, - attn_metadata, - slot_mappings_by_layer, - num_tokens_across_dp=num_tokens_across_dp, - cudagraph_runtime_mode=batch_desc.cg_mode, - ) - - def generate_draft( - self, - num_reqs: int, - num_tokens_padded: int, - attn_metadata: dict[str, Any] | None, - slot_mappings: dict[str, torch.Tensor] | None, - num_tokens_across_dp: torch.Tensor | None, - cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, - ) -> None: - idx_mapping = self.idx_mapping[:num_reqs] - positions = self.input_buffers.positions[:num_reqs] - # Run the eagle model forward pass. - last_hidden_states, hidden_states = self.run_model( - num_tokens_padded, - attn_metadata, - slot_mappings, - num_tokens_across_dp, - cudagraph_runtime_mode, - ) - last_hidden_states = last_hidden_states[:num_reqs] - - # Sample the draft tokens. - logits = self.model.compute_logits(last_hidden_states) - draft_tokens = self._sample_draft( - logits, - idx_mapping, - positions, - self.current_draft_step, - self.draft_logits, - ) - - # Update the inputs for the next step. - update_eagle_draft_inputs( - draft_tokens, - self.current_draft_step, - hidden_states, - self.draft_tokens, - self.hidden_states, - self.input_buffers, - num_reqs, - self.max_model_len, - self.num_speculative_steps, - ) - - def _build_draft_attn_metadata( - self, - num_reqs: int, - num_reqs_padded: int, - num_tokens_padded: int, - ) -> dict[str, Any] | None: - if not self.draft_attn_layer_names: - return None - - query_start_loc_cpu = torch.clamp( - self.arange[: num_reqs_padded + 1], max=num_reqs - ) - block_tables = [ - x[:num_reqs_padded] for x in self.block_tables.input_block_tables - ] - slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded] - attn_metadata = build_attn_metadata( - attn_groups=self.attn_groups, - num_reqs=num_reqs_padded, - num_tokens=num_tokens_padded, - query_start_loc_gpu=self.input_buffers.query_start_loc[ - : num_reqs_padded + 1 - ], - query_start_loc_cpu=query_start_loc_cpu, - max_query_len=1, - seq_lens=self.input_buffers.seq_lens[:num_reqs_padded], - max_seq_len=self.draft_max_seq_len, - block_tables=block_tables, - slot_mappings=slot_mappings, - kv_cache_config=self.kv_cache_config, - ) - return attn_metadata - - def capture( - self, - attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], - ) -> None: - logger.info("Capturing model for Eagle speculator...") - # Reset indices to zeros to prevent stale values from prior - # dummy runs to cause out-of-bounds indexing during capture. - self.last_token_indices.zero_() - - # Capture the prefill routine (model forward + compute_logits + - # sample). - # For FULL graphs, the entire routine is recorded as one graph. - # For PIECEWISE, only the model's compiled regions are captured - # and the rest (compute_logits, gumbel_sample) runs eagerly. - assert self.prefill_cudagraph_manager is not None - if self.prefill_cudagraph_manager.use_breakable_cg: - self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model) - self.prefill_cudagraph_manager.capture( - self.prefill, - attn_states, - progress_bar_desc="Capturing eagle prefill CUDA graphs", - ) - - if self.num_speculative_steps == 1: - return - - # Capture the decode draft generation routine (model forward + - # compute_logits + sample + update_eagle_inputs) for a single - # step. - assert self.decode_cudagraph_manager is not None - self.decode_cudagraph_manager.capture( - self.generate_draft, - self.model_state, - self.input_buffers, - self.block_tables, - self.attn_groups, - self.kv_cache_config, - progress_bar_desc="Capturing eagle decode CUDA graphs", - ) - - @torch.inference_mode() - def propose( - self, - input_batch: InputBatch, - attn_metadata: dict[str, Any], - slot_mappings: dict[str, torch.Tensor], - # [num_tokens, hidden_size] - last_hidden_states: torch.Tensor, - # num_layers x [num_tokens, hidden_size] - aux_hidden_states: list[torch.Tensor] | None, - # [num_reqs] - num_sampled: torch.Tensor, - # [num_reqs] - num_rejected: torch.Tensor, - # [max_num_reqs] - last_sampled: torch.Tensor, - # [max_num_reqs] - next_prefill_tokens: torch.Tensor, - # [max_num_reqs] - temperature: torch.Tensor, - # [max_num_reqs] - seeds: torch.Tensor, - num_tokens_across_dp: torch.Tensor | None = None, - dummy_run: bool = False, - skip_attn_for_dummy_run: bool = False, - mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, - is_profile: bool = False, - ) -> torch.Tensor: - num_tokens = input_batch.num_tokens_after_padding - num_reqs = input_batch.num_reqs - max_query_len = input_batch.num_scheduled_tokens.max() - max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item() - self.draft_max_seq_len = min( - max_seq_len + self.num_speculative_steps, self.max_model_len - ) - - # NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the - # number of rejected tokens, we maintain the size of eagle's input_ids and - # hidden_states the same as the target model's. This means, we pad each - # request's query length to include any rejected positions. By doing so, - # we can also reuse the attention metadata (e.g., query_start_loc, - # seq_lens) of the target model. - if aux_hidden_states: - assert self.method == "eagle3" - hidden_states = self.model.combine_hidden_states( - torch.cat(aux_hidden_states, dim=-1) - ) - else: - hidden_states = last_hidden_states - self.hidden_states[:num_tokens].copy_(hidden_states) - - # Copy temperature, seeds, and idx mapping to the pre-allocated buffers. - # NOTE(woosuk): For draft sampling, we only consider the temperature - # and ignore the other sampling parameters such as top_k and top_p, - # for simplicity and performance. - # While this may slightly degrade the acceptance rate, it does not - # affect the output distribution after rejection sampling. - self.temperature.copy_(temperature) - self.seeds.copy_(seeds) - self.idx_mapping[:num_reqs].copy_(input_batch.idx_mapping) - - # Get the input ids and last token indices for the speculator. - prepare_eagle_inputs( - self.last_token_indices, - self.current_draft_step, - self.input_buffers, - input_batch, - num_sampled, - num_rejected, - last_sampled, - next_prefill_tokens, - self.max_num_reqs, - ) - - # When all requests are decoding (no true prefills), each has - # num_speculative_steps + 1 tokens, enabling FULL graph replay. - uniform_token_count = get_uniform_token_count( - num_reqs, - # Use the actual number of tokens without padding added by - # the target model during FULL cudagraph. - input_batch.num_tokens, - max_query_len, - ) - prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( - self.prefill_cudagraph_manager, - num_reqs, - num_tokens, - uniform_token_count, - dp_size=self.dp_size, - dp_rank=self.dp_rank, - need_eager=is_profile, - ) - - if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL: - # Replay the full graph for draft prefill. - assert self.prefill_cudagraph_manager is not None - self.prefill_cudagraph_manager.run_fullgraph(prefill_batch_desc) - else: - # The target model's attention metadata and slot mappings - # can directly be used for draft prefill, because of the - # identical batch shape and KV cache layout. - self.prefill( - num_reqs, - prefill_batch_desc.num_tokens, - attn_metadata, - slot_mappings, - num_tokens_across_dp=num_tokens_across_dp, - cudagraph_runtime_mode=prefill_batch_desc.cg_mode, - mm_inputs=mm_inputs, - ) - - if self.num_speculative_steps == 1: - # Early exit. - return self.draft_tokens[:num_reqs, :1] - - # Prepare the inputs for the decode steps. - prepare_eagle_decode( - self.draft_tokens[:num_reqs, 0], - input_batch.seq_lens, - num_rejected, - self.input_buffers, - self.max_model_len, - self.max_num_reqs, - ) - - # Each request produces exactly 1 token per draft generation step, - # enabling FULL graph replay. - decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( - self.decode_cudagraph_manager, - num_reqs, - num_reqs, - uniform_token_count=1, - dp_size=self.dp_size, - dp_rank=self.dp_rank, - need_eager=is_profile, - ) - - # Generate the remaining num_speculative_steps - 1 draft tokens. - self.multi_step_decode( - num_reqs, - dummy_run and skip_attn_for_dummy_run, - decode_batch_desc, - num_tokens_across_dp, - ) - - return self.draft_tokens[:num_reqs] - - -@triton.jit -def _prepare_eagle_inputs_kernel( - last_token_indices_ptr, - eagle_current_draft_step_ptr, - eagle_input_ids_ptr, - eagle_positions_ptr, - eagle_query_start_loc_ptr, - eagle_seq_lens_ptr, - target_input_ids_ptr, - target_positions_ptr, - idx_mapping_ptr, - last_sampled_ptr, - next_prefill_tokens_ptr, - num_sampled_ptr, - num_rejected_ptr, - query_start_loc_ptr, - seq_lens_ptr, - max_num_reqs, - BLOCK_SIZE: tl.constexpr, -): - req_idx = tl.program_id(0) - num_reqs = tl.num_programs(0) - req_state_idx = tl.load(idx_mapping_ptr + req_idx) - - query_start = tl.load(query_start_loc_ptr + req_idx) - query_end = tl.load(query_start_loc_ptr + req_idx + 1) - query_len = query_end - query_start - seq_len = tl.load(seq_lens_ptr + req_idx) - - # Get the true query length and next token after accounting for rejected tokens. - num_rejected = tl.load(num_rejected_ptr + req_idx) - query_len -= num_rejected - - num_sampled = tl.load(num_sampled_ptr + req_idx) - if num_sampled > 0: - next_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32) - else: - # Chunked prefilling. - # Get the next prefill token. - next_token = tl.load(next_prefill_tokens_ptr + req_state_idx) - - # Shift target_input_ids by one. - for i in range(1, query_len, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < query_len - input_ids = tl.load(target_input_ids_ptr + query_start + block, mask=mask) - tl.store(eagle_input_ids_ptr + query_start + block - 1, input_ids, mask=mask) - - last_token_index = query_start + query_len - 1 - tl.store(last_token_indices_ptr + req_idx, last_token_index) - tl.store(eagle_input_ids_ptr + last_token_index, next_token) - - # Copy positions. - for i in range(0, query_len, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < query_len - target_pos = tl.load(target_positions_ptr + query_start + block, mask=mask) - tl.store(eagle_positions_ptr + query_start + block, target_pos, mask=mask) - - # Copy query start locations. - tl.store(eagle_query_start_loc_ptr + req_idx, query_start) - # Copy sequence lengths. - tl.store(eagle_seq_lens_ptr + req_idx, seq_len) - if req_idx == (num_reqs - 1): - # Reset the current draft step to 0. - tl.store(eagle_current_draft_step_ptr, 0) - # Pad query_start_loc for CUDA graphs. - for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < max_num_reqs + 1 - tl.store(eagle_query_start_loc_ptr + block, query_end, mask=mask) - # Pad seq_lens for CUDA graphs. - for i in range(num_reqs, max_num_reqs, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < max_num_reqs - tl.store(eagle_seq_lens_ptr + block, 0, mask=mask) - # Pad last_token_indices for CUDA graphs. - for i in range(num_reqs, max_num_reqs, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < max_num_reqs - tl.store(last_token_indices_ptr + block, 0, mask=mask) - - -def prepare_eagle_inputs( - # [num_reqs] - last_token_indices: torch.Tensor, - current_draft_step: torch.Tensor, - input_buffers: InputBuffers, - input_batch: InputBatch, - # [num_reqs] - num_sampled: torch.Tensor, - # [num_reqs] - num_rejected: torch.Tensor, - # [max_num_reqs] - last_sampled: torch.Tensor, - # [max_num_reqs] - next_prefill_tokens: torch.Tensor, - max_num_reqs, -) -> torch.Tensor: - num_reqs = input_batch.num_reqs - _prepare_eagle_inputs_kernel[(num_reqs,)]( - last_token_indices, - current_draft_step, - input_buffers.input_ids, - input_buffers.positions, - input_buffers.query_start_loc, - input_buffers.seq_lens, - input_batch.input_ids, - input_batch.positions, - input_batch.idx_mapping, - last_sampled, - next_prefill_tokens, - num_sampled, - num_rejected, - input_batch.query_start_loc, - input_batch.seq_lens, - max_num_reqs, - BLOCK_SIZE=1024, - ) - return last_token_indices - - -@triton.jit -def _prepare_eagle_decode_kernel( - draft_tokens_ptr, - draft_tokens_stride, - target_seq_lens_ptr, - num_rejected_ptr, - input_ids_ptr, - positions_ptr, - query_start_loc_ptr, - seq_lens_ptr, - max_model_len, - max_num_reqs, - BLOCK_SIZE: tl.constexpr, -): - req_idx = tl.program_id(0) - num_reqs = tl.num_programs(0) - 1 - if req_idx == num_reqs: - # Compute query_start_loc. Pad it with the last query_start_loc - # for CUDA graphs. - for i in range(0, max_num_reqs + 1, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - q = tl.where(block < num_reqs, block, num_reqs) - mask = block < max_num_reqs + 1 - tl.store(query_start_loc_ptr + block, q, mask=mask) - # Pad seq_lens for CUDA graphs. - for i in range(req_idx, max_num_reqs, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < max_num_reqs - tl.store(seq_lens_ptr + block, 0, mask=mask) - return - - # draft token -> input id. - draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride) - tl.store(input_ids_ptr + req_idx, draft_token) - - # Compute position and seq_lens. - # NOTE(woosuk): To prevent out-of-range access, we clamp these values - # if they reach the max model length. - position = tl.load(positions_ptr + req_idx) - position = tl.minimum(position + 1, max_model_len - 1) - tl.store(positions_ptr + req_idx, position) - - target_seq_len = tl.load(target_seq_lens_ptr + req_idx) - num_rejected = tl.load(num_rejected_ptr + req_idx) - seq_len = target_seq_len - num_rejected - seq_len = tl.minimum(seq_len + 1, max_model_len) - tl.store(seq_lens_ptr + req_idx, seq_len) - - -def prepare_eagle_decode( - draft_tokens: torch.Tensor, - target_seq_lens: torch.Tensor, - num_rejected: torch.Tensor, - input_buffers: InputBuffers, - max_model_len: int, - max_num_reqs: int, -): - num_reqs = draft_tokens.shape[0] - _prepare_eagle_decode_kernel[(num_reqs + 1,)]( - draft_tokens, - draft_tokens.stride(0), - target_seq_lens, - num_rejected, - input_buffers.input_ids, - input_buffers.positions, - input_buffers.query_start_loc, - input_buffers.seq_lens, - max_model_len, - max_num_reqs, - BLOCK_SIZE=1024, - ) - - -@triton.jit -def _update_eagle_draft_inputs_kernel( - output_draft_tokens_ptr, - output_draft_tokens_stride, - next_input_hidden_states_ptr, - next_input_hidden_states_stride, - input_ids_ptr, - positions_ptr, - seq_lens_ptr, - draft_tokens_ptr, - current_draft_step_ptr, - hidden_states_ptr, - hidden_states_stride, - hidden_size, - max_model_len, - num_speculative_steps, - BLOCK_SIZE: tl.constexpr, -): - req_idx = tl.program_id(0) - - # Write the sampled draft token into self.draft_tokens[req_idx, step]. - draft_token = tl.load(draft_tokens_ptr + req_idx) - step = tl.load(current_draft_step_ptr) - tl.store( - output_draft_tokens_ptr + req_idx * output_draft_tokens_stride + step, - draft_token, - ) - - if step >= num_speculative_steps - 1: - # This is the final step. Skip updating draft forward inputs. - return - - # Write the sampled draft token into the input ids tensor for the next - # forward pass. - tl.store(input_ids_ptr + req_idx, draft_token) - - # Copy hidden states into the input hidden states tensor for the next - # forward pass. - for i in range(0, hidden_size, BLOCK_SIZE): - block = i + tl.arange(0, BLOCK_SIZE) - mask = block < hidden_size - hidden_states = tl.load( - hidden_states_ptr + req_idx * hidden_states_stride + block, - mask=mask, - ) - tl.store( - next_input_hidden_states_ptr - + req_idx * next_input_hidden_states_stride - + block, - hidden_states, - mask=mask, - ) - - # Increment position and seq_lens. - # NOTE(woosuk): To prevent out-of-range access, we clamp these values - # if they reach the max model length. - position = tl.load(positions_ptr + req_idx) - position = tl.minimum(position + 1, max_model_len - 1) - tl.store(positions_ptr + req_idx, position) - - seq_len = tl.load(seq_lens_ptr + req_idx) - seq_len = tl.minimum(seq_len + 1, max_model_len) - tl.store(seq_lens_ptr + req_idx, seq_len) - - -def update_eagle_draft_inputs( - draft_tokens: torch.Tensor, - current_draft_step: torch.Tensor, - hidden_states: torch.Tensor, - output_draft_tokens: torch.Tensor, - next_input_hidden_states: torch.Tensor, - input_buffers: InputBuffers, - num_reqs: int, - max_model_len: int, - num_speculative_steps: int, -): - _, hidden_size = hidden_states.shape - _update_eagle_draft_inputs_kernel[(num_reqs,)]( - output_draft_tokens, - output_draft_tokens.stride(0), - next_input_hidden_states, - next_input_hidden_states.stride(0), - input_buffers.input_ids, - input_buffers.positions, - input_buffers.seq_lens, - draft_tokens, - current_draft_step, - hidden_states, - hidden_states.stride(0), - hidden_size, - max_model_len, - num_speculative_steps, - BLOCK_SIZE=1024, - ) + target_model: nn.Module, + target_attn_layer_names: set[str], + ) -> nn.Module: + return load_eagle_model(target_model, self.vllm_config) diff --git a/vllm/v1/worker/gpu/spec_decode/gemma4/__init__.py b/vllm/v1/worker/gpu/spec_decode/gemma4/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/gemma4/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py new file mode 100644 index 0000000000..fcbea5d101 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Gemma4 MTP (Multi-Token Prediction) speculator for speculative decoding. + +The Gemma4 assistant model runs all decoder layers per draft step +(producing one token), and all its attention layers share KV cache +with the target model via cross-model KV sharing. +""" + +from collections import defaultdict + +import torch.nn as nn + +from vllm.compilation.backends import set_model_tag +from vllm.config import VllmConfig, replace +from vllm.distributed.parallel_state import get_pp_group +from vllm.logger import init_logger +from vllm.model_executor.model_loader import get_model +from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( + AutoRegressiveSpeculator, +) + +logger = init_logger(__name__) + + +class Gemma4Speculator(AutoRegressiveSpeculator): + @property + def advance_draft_positions(self) -> bool: + # Gemma4 MTP is Q-only and reads K/V from the target's existing cache. + # No new KV slots are written, so positions and seq_lens stay fixed. + return False + + @property + def model_returns_tuple(self) -> bool: + # forward() returns (draft_hidden_states, backbone_hidden_states). + # The proposer uses draft_hidden_states for compute_logits and + # backbone_hidden_states for the hidden-state feedback buffer. + return True + + def load_draft_model( + self, + target_model: nn.Module, + target_attn_layer_names: set[str], + ) -> nn.Module: + draft_vllm_config = self._create_draft_vllm_config() + with set_model_tag("eagle_head"): + draft_model = get_model( + vllm_config=draft_vllm_config, + model_config=self.speculative_config.draft_model_config, + load_config=self.speculative_config.draft_load_config, + ) + self._setup_gemma4_kv_sharing(draft_model, target_attn_layer_names) + self._share_embeddings(draft_model, target_model) + return draft_model + + def _create_draft_vllm_config(self) -> VllmConfig: + """Preserve the target's forced TRITON_ATTN backend for draft layers. + + Gemma4 forces TRITON_ATTN due to heterogeneous head dimensions + (head_dim=256 sliding, global_head_dim=512 full). The base class + resets attention_config.backend to None for draft models, causing + sliding layers to fall back to FLASH_ATTN which cannot handle + KV-shared cache. Override to carry the target's backend through. + """ + draft_model_config = self.speculative_config.draft_model_config + draft_vllm_config = replace( + self.vllm_config, + model_config=draft_model_config, + ) + target_backend = self.vllm_config.attention_config.backend + if target_backend is not None: + draft_vllm_config = replace( + draft_vllm_config, + attention_config=replace( + draft_vllm_config.attention_config, + backend=target_backend, + ), + ) + return draft_vllm_config + + def _setup_gemma4_kv_sharing( + self, + model: nn.Module, + target_attn_layer_names: set[str], + ) -> None: + """Wire draft layers to share KV with the target model. + + Each draft decoder layer is mapped to the last non-KV-shared + target layer of the same attention type (sliding or full). + """ + draft_config = self.speculative_config.draft_model_config.hf_config + draft_text_config = draft_config.get_text_config() + target_config = self.vllm_config.model_config.hf_config + target_text_config = target_config.get_text_config() + target_layer_types = getattr(target_text_config, "layer_types", []) + + if not (hasattr(model, "model") and hasattr(model.model, "layers")): + return + + target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0) + num_non_shared = len(target_layer_types) - target_num_kv_shared + type_to_target_indices: dict[str, list[int]] = defaultdict(list) + for idx, lt in enumerate(target_layer_types[:num_non_shared]): + type_to_target_indices[lt].append(idx) + + target_prefix = "model.layers" + for name in target_attn_layer_names: + if ".layers." in name: + target_prefix = name.split(".layers.")[0] + ".layers" + break + + draft_layer_types = getattr(draft_text_config, "layer_types", []) + for draft_idx, layer in enumerate(model.model.layers): + if not hasattr(layer, "self_attn"): + continue + attn = getattr(layer.self_attn, "attn", None) + if attn is None: + continue + + draft_layer_type = ( + draft_layer_types[draft_idx] + if draft_idx < len(draft_layer_types) + else "full_attention" + ) + candidates = type_to_target_indices.get(draft_layer_type, []) + if not candidates: + logger.warning( + "No target layer of type '%s' for draft layer %d", + draft_layer_type, + draft_idx, + ) + continue + + target_idx = candidates[-1] + target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn" + attn.kv_sharing_target_layer_name = target_layer_name + logger.info( + "Gemma4 MTP: draft layer %d (%s) -> %s", + draft_idx, + draft_layer_type, + target_layer_name, + ) + + def _share_embeddings( + self, + draft_model: nn.Module, + target_model: nn.Module, + ) -> None: + target_language_model = ( + target_model.get_language_model() + if hasattr(target_model, "get_language_model") + else target_model + ) + if get_pp_group().world_size == 1: + target_embed = getattr(target_language_model.model, "embed_tokens", None) + if target_embed is not None: + del draft_model.model.embed_tokens + draft_model.model.embed_tokens = target_embed diff --git a/vllm/v1/worker/gpu/spec_decode/mtp/__init__.py b/vllm/v1/worker/gpu/spec_decode/mtp/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/mtp/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py new file mode 100644 index 0000000000..e6abb0be83 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch.nn as nn + +from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( + AutoRegressiveSpeculator, +) +from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model + + +class MTPSpeculator(AutoRegressiveSpeculator): + @property + def model_returns_tuple(self) -> bool: + return False + + def load_draft_model( + self, + target_model: nn.Module, + target_attn_layer_names: set[str], + ) -> nn.Module: + return load_eagle_model(target_model, self.vllm_config) diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py new file mode 100644 index 0000000000..e8fa8af53b --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from abc import ABC, abstractmethod +from typing import Any + +import torch +import torch.nn as nn + +from vllm.config import VllmConfig, get_layers_from_vllm_config +from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.attn_utils import ( + build_attn_metadata, + init_attn_backend, +) +from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.cudagraph_utils import ( + AttentionStatePair, + BatchExecutionDescriptor, +) +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.interface import ModelState + + +class BaseSpeculator(ABC): + @abstractmethod + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: + pass + + @abstractmethod + def capture( + self, + attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], + ) -> None: + pass + + @abstractmethod + def propose( + self, + input_batch: InputBatch, + attn_metadata: dict[str, Any], + slot_mappings: dict[str, torch.Tensor], + # [num_tokens, hidden_size] + last_hidden_states: torch.Tensor, + # num_layers x [num_tokens, hidden_size] + aux_hidden_states: list[torch.Tensor] | None, + # [num_reqs] + num_sampled: torch.Tensor, + # [num_reqs] + num_rejected: torch.Tensor, + # [max_num_reqs] + last_sampled: torch.Tensor, + # [max_num_reqs] + next_prefill_tokens: torch.Tensor, + # [max_num_reqs] + temperature: torch.Tensor, + # [max_num_reqs] + seeds: torch.Tensor, + num_tokens_across_dp: torch.Tensor | None = None, + dummy_run: bool = False, + skip_attn_for_dummy_run: bool = False, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + is_profile: bool = False, + ) -> torch.Tensor: + pass + + +class DraftModelSpeculator(BaseSpeculator): + def __init__(self, vllm_config: VllmConfig, device: torch.device): + self.vllm_config = vllm_config + self.device = device + + assert vllm_config.speculative_config is not None + self.speculative_config = vllm_config.speculative_config + self.method = self.speculative_config.method + self.num_speculative_steps = self.speculative_config.num_speculative_tokens + self.draft_model_config = self.speculative_config.draft_model_config + + self.scheduler_config = vllm_config.scheduler_config + self.max_num_reqs = self.scheduler_config.max_num_seqs + self.max_num_tokens = self.scheduler_config.max_num_batched_tokens + self.max_model_len = vllm_config.model_config.max_model_len + self.draft_max_seq_len = self.max_model_len + # We need to get the hidden size from the draft model config because + # the draft model's hidden size can be different from the target model's + # hidden size (e.g., Llama 3.3 70B). + self.hidden_size = self.draft_model_config.get_hidden_size() + # Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP + # draft the target's pre-hc_head (T, hc_mult * hidden_size) residual). + # Non-HC models default to hc_mult=1 and are unaffected. + hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1) + self.hidden_size = self.hidden_size * hc_mult + self.vocab_size = self.draft_model_config.get_vocab_size() + self.dtype = vllm_config.model_config.dtype + self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel + + # DP configuration + self.dp_size = vllm_config.parallel_config.data_parallel_size + self.dp_rank = vllm_config.parallel_config.data_parallel_rank + + self.input_buffers = InputBuffers( + max_num_reqs=self.max_num_reqs, + max_num_tokens=self.max_num_tokens, + device=device, + ) + self.idx_mapping = torch.zeros( + self.max_num_reqs, dtype=torch.int32, device=device + ) + self.temperature = torch.zeros( + self.max_num_reqs, dtype=torch.float32, device=device + ) + self.seeds = torch.zeros(self.max_num_reqs, dtype=torch.int64, device=device) + self.draft_tokens = torch.zeros( + self.max_num_reqs, + self.num_speculative_steps, + dtype=torch.int64, + device=device, + ) + self.arange = torch.arange( + self.max_num_reqs + 1, dtype=torch.int32, device="cpu" + ) + + self.draft_logits: torch.Tensor | None = None + if self.speculative_config.draft_sample_method == "probabilistic": + self.draft_logits = torch.zeros( + self.max_num_reqs, + self.num_speculative_steps, + self.vocab_size, + dtype=torch.float32, + device=device, + ) + + @abstractmethod + def load_draft_model( + self, + target_model: nn.Module, + target_attn_layer_names: set[str], + ) -> nn.Module: + pass + + def load_model(self, target_model: nn.Module) -> None: + target_attn_layer_names = set( + get_layers_from_vllm_config( + self.vllm_config, + AttentionLayerBase, # type: ignore[type-abstract] + ).keys() + ) + + self.model = self.load_draft_model(target_model, target_attn_layer_names) + + all_attn_layers = set[str]( + get_layers_from_vllm_config( + self.vllm_config, + AttentionLayerBase, # type: ignore[type-abstract] + ).keys() + ) + self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names + + def set_attn( + self, + model_state: ModelState, + kv_cache_config: KVCacheConfig, + block_tables: BlockTables, + ) -> None: + self.model_state = model_state + self.kv_cache_config = kv_cache_config + self.attn_groups, _, _ = init_attn_backend( + kv_cache_config, + self.vllm_config, + self.device, + active_layer_names=self.draft_attn_layer_names, + ) + self.block_tables = block_tables + + def _build_draft_attn_metadata( + self, + num_reqs: int, + num_reqs_padded: int, + num_tokens_padded: int, + ) -> dict[str, Any] | None: + query_start_loc_cpu = torch.clamp( + self.arange[: num_reqs_padded + 1], max=num_reqs + ) + block_tables = [ + x[:num_reqs_padded] for x in self.block_tables.input_block_tables + ] + slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded] + attn_metadata = build_attn_metadata( + attn_groups=self.attn_groups, + num_reqs=num_reqs_padded, + num_tokens=num_tokens_padded, + query_start_loc_gpu=self.input_buffers.query_start_loc[ + : num_reqs_padded + 1 + ], + query_start_loc_cpu=query_start_loc_cpu, + max_query_len=1, + seq_lens=self.input_buffers.seq_lens[:num_reqs_padded], + max_seq_len=self.draft_max_seq_len, + block_tables=block_tables, + slot_mappings=slot_mappings, + kv_cache_config=self.kv_cache_config, + ) + return attn_metadata + + def _copy_request_inputs( + self, + num_reqs: int, + # [num_reqs] + idx_mapping: torch.Tensor, + # [max_num_reqs] + temperature: torch.Tensor, + # [max_num_reqs] + seeds: torch.Tensor, + ) -> None: + # Copy temperature, seeds, and idx mapping to the pre-allocated buffers. + # NOTE(woosuk): For draft sampling, we only consider the temperature + # and ignore the other sampling parameters such as top_k and top_p, + # for simplicity and performance. + # While this may slightly degrade the acceptance rate, it does not + # affect the output distribution after rejection sampling. + self.temperature.copy_(temperature) + self.seeds.copy_(seeds) + self.idx_mapping[:num_reqs].copy_(idx_mapping)