From 0bae1d38480374365ad77bbea50be225237572ea Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Wed, 10 Jun 2026 11:47:46 -0400 Subject: [PATCH] [MRV2][Spec Decode] DFlash (#44586) Signed-off-by: Giancarlo Delfin Signed-off-by: Benjamin Chislett Signed-off-by: Benjamin Chislett Co-authored-by: Giancarlo Delfin --- tests/v1/e2e/spec_decode/test_spec_decode.py | 14 +- vllm/config/vllm.py | 12 +- vllm/v1/worker/gpu/attn_utils.py | 3 +- vllm/v1/worker/gpu/model_runner.py | 9 +- vllm/v1/worker/gpu/sample/gumbel.py | 13 +- vllm/v1/worker/gpu/spec_decode/__init__.py | 8 +- .../spec_decode/autoregressive/speculator.py | 29 - .../worker/gpu/spec_decode/dflash/__init__.py | 0 .../gpu/spec_decode/dflash/cudagraph.py | 114 ++++ .../gpu/spec_decode/dflash/speculator.py | 573 ++++++++++++++++++ .../v1/worker/gpu/spec_decode/dflash/utils.py | 70 +++ .../gpu/spec_decode/eagle/eagle3_utils.py | 9 +- vllm/v1/worker/gpu/spec_decode/speculator.py | 42 +- vllm/v1/worker/gpu/spec_decode/utils.py | 19 + 14 files changed, 868 insertions(+), 47 deletions(-) create mode 100644 vllm/v1/worker/gpu/spec_decode/dflash/__init__.py create mode 100644 vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py create mode 100644 vllm/v1/worker/gpu/spec_decode/dflash/speculator.py create mode 100644 vllm/v1/worker/gpu/spec_decode/dflash/utils.py diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index a9092bb766..06e8b3bf0e 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -1300,13 +1300,18 @@ def dflash_config(): ) -def test_dflash_acceptance_rates(dflash_config): +@pytest.mark.parametrize("use_mrv2", [False, True]) +def test_dflash_acceptance_rates( + monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config +): """ E2E test for DFlash (block diffusion) speculative decoding. Runs acceptance rate validation on GSM8k, MT-Bench, and HumanEval comparing against baseline results from the paper (Table 1). See https://github.com/z-lab/dflash/blob/main/benchmark_sglang.py for methodology. """ + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0") + spec_llm = LLM(**dflash_config) max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k @@ -1414,11 +1419,16 @@ def test_synthetic_acceptance_rate(): cleanup_dist_env_and_memory() -def test_dflash_correctness(dflash_config): +@pytest.mark.parametrize("use_mrv2", [False, True]) +def test_dflash_correctness( + monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config +): """ E2E test for DFlash (block diffusion) speculative decoding. Ensures output correctness on GSM8k, with cudagraphs and batching on. """ + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0") + spec_llm = LLM(**dflash_config) # Evaluate GSM8k accuracy (Qwen3-8B ref: ~87-92% on GSM8k) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index fee1d20350..9be5638132 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2008,12 +2008,16 @@ class VllmConfig: # TODO: ngram / ngram_gpu are not supported by the v2 model runner yet if speculative_config.method in ("ngram", "ngram_gpu"): unsupported.append("ngram/ngram_gpu speculative decoding") - elif speculative_config.method not in ("eagle", "eagle3", "mtp"): + elif speculative_config.method not in ("eagle", "eagle3", "mtp", "dflash"): unsupported.append(f"speculative method '{speculative_config.method}'") - # V2 EagleSpeculator does not support parallel_drafting (required by PEagle) - if speculative_config.parallel_drafting: - unsupported.append("parallel drafting for speculative decoding") + # V2 EagleSpeculator does not support parallel_drafting (for P-Eagle) + # DFlash uses parallel drafting natively in V2 via DFlashSpeculator. + if ( + speculative_config.parallel_drafting + and speculative_config.method != "dflash" + ): + unsupported.append("parallel drafting for EAGLE speculative decoding") if ( speculative_config.method == "eagle3" diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 6fc55ee320..35c40a1c22 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -394,6 +394,7 @@ def build_attn_metadata( positions: torch.Tensor | None = None, model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, for_cudagraph_capture: bool = False, + causal: bool = True, ) -> dict[str, Any]: seq_lens = seq_lens[:num_reqs] if dcp_local_seq_lens is not None: @@ -423,7 +424,7 @@ def build_attn_metadata( max_query_len=max_query_len, block_table_tensor=block_table, slot_mapping=slot_mapping, - causal=True, + causal=causal, dcp_local_seq_lens=dcp_local_seq_lens, positions=positions, **common_attn_metadata_extra_kwargs, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 50b78474bb..7cd1e6c5c8 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -193,11 +193,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.is_last_pp_rank: self.speculator = init_speculator(self.vllm_config, self.device) - if self.speculative_config.method == "eagle3": - # EAGLE3 may require auxiliary hidden states from target model outputs. + if self.speculative_config.method in ("eagle3", "dflash"): + # Drafting may require auxiliary hidden states from target model outputs self.use_aux_hidden_state_outputs = True if self.use_pp: - raise ValueError("EAGLE3 with pipeline parallel is not supported.") + raise ValueError( + f"{self.speculative_config.method} with pipeline parallel " + "is not supported." + ) # Draft tokens propagation - for spec-dec + struct outputs. self.draft_tokens_handler = DraftTokensHandler(self.device) diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index aaa49283d3..44d12738cc 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -91,6 +91,7 @@ def gumbel_block_argmax( vocab_size, APPLY_TEMPERATURE: tl.constexpr, USE_FP64: tl.constexpr, + PER_TOKEN_COL: tl.constexpr = False, ): req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx) temp = tl.load(temp_ptr + req_state_idx).to(tl.float32) @@ -103,7 +104,10 @@ def gumbel_block_argmax( if processed_logits_ptr is not None: # Store the temperature-applied logits. if processed_logits_col_ptr is not None: - col = tl.load(processed_logits_col_ptr) + if PER_TOKEN_COL: + col = tl.load(processed_logits_col_ptr + token_idx) + else: + col = tl.load(processed_logits_col_ptr) else: col = 0 tl.store( @@ -158,6 +162,7 @@ def _gumbel_sample_kernel( BLOCK_SIZE: tl.constexpr, APPLY_TEMPERATURE: tl.constexpr, USE_FP64: tl.constexpr, + PER_TOKEN_COL: tl.constexpr, ): token_idx = tl.program_id(0) block_idx = tl.program_id(1) @@ -185,6 +190,7 @@ def _gumbel_sample_kernel( vocab_size, APPLY_TEMPERATURE=APPLY_TEMPERATURE, USE_FP64=USE_FP64, + PER_TOKEN_COL=PER_TOKEN_COL, ) token_id = block_idx * BLOCK_SIZE + idx tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id) @@ -208,6 +214,10 @@ def gumbel_sample( local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64) local_max_dtype = torch.float64 if use_fp64 else torch.float32 local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype) + per_token_col = ( + output_processed_logits_col is not None + and output_processed_logits_col.dim() > 0 + ) _gumbel_sample_kernel[(num_tokens, num_blocks)]( local_argmax, local_argmax.stride(0), @@ -226,6 +236,7 @@ def gumbel_sample( BLOCK_SIZE=BLOCK_SIZE, APPLY_TEMPERATURE=apply_temperature, USE_FP64=use_fp64, + PER_TOKEN_COL=per_token_col, ) # NOTE(woosuk): Use int64 for later indexing. max_block_idx = local_max.argmax(dim=-1, keepdim=True) diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index bafb28c5cc..09153dd20f 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -8,7 +8,13 @@ from vllm.config import VllmConfig def init_speculator(vllm_config: VllmConfig, device: torch.device): speculative_config = vllm_config.speculative_config assert speculative_config is not None - if speculative_config.use_gemma4_mtp(): + if speculative_config.method == "dflash": + from vllm.v1.worker.gpu.spec_decode.dflash.speculator import ( + DFlashSpeculator, + ) + + return DFlashSpeculator(vllm_config, device) + elif speculative_config.use_gemma4_mtp(): from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import ( Gemma4Speculator, ) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 868540437b..775c06f7b8 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -18,7 +18,6 @@ from vllm.v1.worker.gpu.cudagraph_utils import ( ) from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers -from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( DecodeSpeculatorCudaGraphManager, PrefillSpeculatorCudaGraphManager, @@ -279,34 +278,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): return self.draft_tokens[:num_reqs] - def sample_draft( - self, - hidden_states: torch.Tensor, - positions: torch.Tensor, - idx_mapping: torch.Tensor, - temperature: torch.Tensor, - seeds: torch.Tensor, - draft_step: torch.Tensor, - draft_logits: torch.Tensor | None, - ) -> torch.Tensor: - logits = self.model.compute_logits(hidden_states) - if draft_logits is not None: - # NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise - # used for draft and target sampling. - return gumbel_sample( - logits, - idx_mapping, - temperature, - seeds, - positions + 1, - apply_temperature=True, - output_processed_logits=draft_logits, - output_processed_logits_col=draft_step, - use_fp64=self.use_fp64_gumbel, - ) - else: - return logits.argmax(dim=-1) - @torch.inference_mode() def _run_model( self, diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/__init__.py b/vllm/v1/worker/gpu/spec_decode/dflash/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py new file mode 100644 index 0000000000..3e4b2b7e7f --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable + +import torch + +from vllm.config.compilation import CUDAGraphMode +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.attn_utils import ( + build_attn_metadata, + build_slot_mappings_by_layer, +) +from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.cudagraph_utils import ( + AttentionState, + BatchExecutionDescriptor, + CudaGraphManager, +) +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.utils import AttentionGroup + + +def _prepare_dflash_inputs_to_capture( + num_reqs: int, + num_tokens: int, + input_buffers: InputBuffers, + block_tables: BlockTables, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + max_model_len: int, + skip_attn: bool, + causal: bool, +) -> AttentionState: + input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) + input_block_tables = block_tables.get_dummy_block_tables(num_reqs) + slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens) + slot_mappings_by_layer = build_slot_mappings_by_layer( + slot_mappings, kv_cache_config + ) + + attn_metadata = None + if not skip_attn: + query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) + attn_metadata = build_attn_metadata( + attn_groups=attn_groups, + num_reqs=num_reqs, + num_tokens=num_tokens, + query_start_loc_gpu=input_batch.query_start_loc, + query_start_loc_cpu=query_start_loc_cpu, + max_query_len=num_tokens // num_reqs, + seq_lens=input_batch.seq_lens, + max_seq_len=max_model_len, + block_tables=input_block_tables, + slot_mappings=slot_mappings, + kv_cache_config=kv_cache_config, + for_cudagraph_capture=True, + causal=causal, + ) + return AttentionState(attn_metadata, slot_mappings_by_layer) + + +class DFlashCudaGraphManager(CudaGraphManager): + """DFlash CudaGraphManager for the parallel-drafting query forward, + building its own attention metadata from scratch.""" + + def __init__(self, *args, causal: bool = False, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.causal = causal + + def capture( + self, + forward_fn: Callable, + input_buffers: InputBuffers, + block_tables: BlockTables, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + max_model_len: int, + progress_bar_desc: str = "Capturing CUDA graphs", + ) -> None: + def create_forward_fn( + desc: BatchExecutionDescriptor, + warmup: bool, + ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: + num_tokens = desc.num_tokens + num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) + num_tokens_across_dp = ( + torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu") + if self.dp_size > 1 + else None + ) + attn_state = _prepare_dflash_inputs_to_capture( + num_reqs, + num_tokens, + input_buffers, + block_tables, + attn_groups, + kv_cache_config, + max_model_len, + skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE), + causal=self.causal, + ) + attn_metadata, slot_mappings = attn_state + + fwd = lambda cg_mode: forward_fn( + num_reqs, + num_tokens, + attn_metadata, + slot_mappings, + num_tokens_across_dp, + cg_mode, + ) + return fwd, attn_state + + super().capture(create_forward_fn, progress_bar_desc) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py new file mode 100644 index 0000000000..1bd130838a --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -0,0 +1,573 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +import torch +import torch.nn as nn + +from vllm.config import VllmConfig +from vllm.config.compilation import CUDAGraphMode +from vllm.forward_context import BatchDescriptor, set_forward_context +from vllm.logger import init_logger +from vllm.triton_utils import tl, triton +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer +from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager +from vllm.v1.worker.gpu.spec_decode.dflash.utils import ( + get_dflash_causal, + load_dflash_model, +) +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id + +logger = init_logger(__name__) + + +class DFlashSpeculator(DraftModelSpeculator): + def __init__(self, vllm_config: VllmConfig, device: torch.device): + super().__init__(vllm_config, device) + + self.hidden_states = torch.zeros( + self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device + ) + + # Multimodal inputs not currently supported. + self.supports_mm_inputs = False + + # Each request emits exactly (bonus + N mask) query tokens per step. + self.num_query_per_req = 1 + self.num_speculative_steps + + self.parallel_drafting_token_id = get_parallel_drafting_token_id( + self.draft_model_config.hf_config + ) + + self.dflash_causal = get_dflash_causal(self.draft_model_config) + + # Buffers for context K/V precomputation. Populated by prepare_dflash_inputs, + # and processed by the model's precompute_and_store_context_kv method. + # NOT captured by CUDA graphs. + self.context_positions = torch.zeros( + self.max_num_tokens, dtype=torch.int64, device=device + ) + self.context_slot_mapping = torch.zeros( + self.max_num_tokens, dtype=torch.int64, device=device + ) + + # Per-mask-token sampling buffers. Flattened from (num_reqs, num_spec_tokens). + max_num_sampled_tokens = self.max_num_reqs * self.num_speculative_steps + self.sample_indices = torch.zeros( + max_num_sampled_tokens, dtype=torch.int64, device=device + ) + self.sample_pos = torch.zeros( + max_num_sampled_tokens, dtype=torch.int64, device=device + ) + self.sample_idx_mapping = torch.zeros( + max_num_sampled_tokens, dtype=torch.int32, device=device + ) + # [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into + # draft_logits[req, step, :]. + self.sample_col = torch.arange( + self.num_speculative_steps, dtype=torch.int32, device=device + ).repeat(self.max_num_reqs) + + self.query_cudagraph_manager: DFlashCudaGraphManager | None = None + self.draft_kv_cache_group_id: int = -1 + + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: + # PIECEWISE cudagraphs are not supported for dflash + if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL: + cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY + else: + cudagraph_mode = CUDAGraphMode.NONE + + self.query_cudagraph_manager = DFlashCudaGraphManager( + self.vllm_config, + self.device, + cudagraph_mode, + decode_query_len=self.num_query_per_req, + causal=self.dflash_causal, + ) + + def capture(self, attn_states: dict | None = None) -> None: + logger.info("Capturing model for DFlash speculator...") + # Reset sampling indices to zero to prevent stale values from prior + # dummy runs from being baked into the captured graph. + self.sample_indices.zero_() + self.sample_pos.zero_() + self.sample_idx_mapping.zero_() + assert self.query_cudagraph_manager is not None + self.query_cudagraph_manager.capture( + self._generate_draft, + self.input_buffers, + self.block_tables, + self.attn_groups, + self.kv_cache_config, + self.max_model_len, + progress_bar_desc="Capturing dflash CUDA graphs", + ) + + def load_draft_model( + self, + target_model: nn.Module, + target_attn_layer_names: set[str], + ) -> nn.Module: + return load_dflash_model(target_model, self.vllm_config) + + def set_attn( + self, + model_state: ModelState, + kv_cache_config: KVCacheConfig, + block_tables: BlockTables, + ) -> None: + super().set_attn(model_state, kv_cache_config, block_tables) + + # DFlash precomputes context K/V with a single block_size; mixing + # kv-cache groups would silently corrupt the cache for the non-matching group. + draft_groups = [gid for gid, g in enumerate(self.attn_groups) if g] + assert len(draft_groups) == 1, ( + "DFlash currently requires all draft attention layers to share " + "a single kv-cache group." + ) + self.draft_kv_cache_group_id = draft_groups[0] + self.draft_block_size = self.block_tables.block_sizes[ + self.draft_kv_cache_group_id + ] + + @torch.inference_mode() + def _run_model( + self, + num_tokens: int, + attn_metadata: dict[str, Any] | None, + slot_mappings: dict[str, torch.Tensor] | None, + num_tokens_across_dp: torch.Tensor | None, + cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, + ) -> torch.Tensor: + batch_descriptor = BatchDescriptor(num_tokens=num_tokens) + with set_forward_context( + attn_metadata, + self.vllm_config, + num_tokens=num_tokens, + cudagraph_runtime_mode=cudagraph_runtime_mode, + num_tokens_across_dp=num_tokens_across_dp, + slot_mapping=slot_mappings, + batch_descriptor=batch_descriptor, + ): + last_hidden_states = self.model( + input_ids=self.input_buffers.input_ids[:num_tokens], + positions=self.input_buffers.positions[:num_tokens], + inputs_embeds=None, + ) + return last_hidden_states + + def _generate_draft( + self, + num_reqs: int, + num_tokens_padded: int, + attn_metadata: dict[str, Any] | None, + slot_mappings: dict[str, torch.Tensor] | None, + num_tokens_across_dp: torch.Tensor | None, + cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, + ) -> None: + last_hidden_states = self._run_model( + num_tokens_padded, + attn_metadata, + slot_mappings, + num_tokens_across_dp, + cudagraph_runtime_mode, + ) + + num_sample = num_reqs * self.num_speculative_steps + sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]] + draft_tokens = self.sample_draft( + sample_hidden_states, + self.sample_pos[:num_sample], + self.sample_idx_mapping[:num_sample], + self.temperature, + self.seeds, + self.sample_col[:num_sample], + self.draft_logits, + ) + self.draft_tokens[:num_reqs] = draft_tokens.view( + num_reqs, self.num_speculative_steps + ) + + def _build_draft_attn_metadata( + self, + num_reqs: int, + num_reqs_padded: int, + num_tokens_padded: int, + num_query_per_req: int | None = None, + causal: bool = False, + ) -> dict[str, Any] | None: + if not self.draft_attn_layer_names: + return None + assert num_query_per_req is None # Omitted for DFlash, read from self instead + return super()._build_draft_attn_metadata( + num_reqs, + num_reqs_padded, + num_tokens_padded, + num_query_per_req=self.num_query_per_req, + causal=causal, + ) + + @torch.inference_mode() + def propose( + self, + input_batch: InputBatch, + attn_metadata: dict[str, Any], + slot_mappings: dict[str, torch.Tensor], + # [num_tokens, hidden_size] + last_hidden_states: torch.Tensor, + # num_layers x [num_tokens, hidden_size] + aux_hidden_states: list[torch.Tensor] | None, + # [num_reqs] + num_sampled: torch.Tensor, + # [num_reqs] + num_rejected: torch.Tensor, + # [max_num_reqs] + last_sampled: torch.Tensor, + # [max_num_reqs] + next_prefill_tokens: torch.Tensor, + # [max_num_reqs] + temperature: torch.Tensor, + # [max_num_reqs] + seeds: torch.Tensor, + num_tokens_across_dp: torch.Tensor | None = None, + dummy_run: bool = False, + skip_attn_for_dummy_run: bool = False, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + is_profile: bool = False, + ) -> torch.Tensor: + num_reqs = input_batch.num_reqs + num_target_tokens = input_batch.num_tokens + num_query_tokens = num_reqs * self.num_query_per_req + max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item() + self.draft_max_seq_len = min( + max_seq_len + self.num_query_per_req, self.max_model_len + ) + + # NOTE: To avoid CPU-GPU synchronization without CPU knowing the + # number of rejected tokens, we maintain the size of input_ids and + # hidden_states the same as the target model's. This means, we pad each + # request's query length to include any rejected positions. + if aux_hidden_states: + hidden_states = self.model.combine_hidden_states( + torch.cat(aux_hidden_states, dim=-1) + ) + else: + hidden_states = last_hidden_states + self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens]) + + self._copy_request_inputs( + num_reqs, + input_batch.idx_mapping, + temperature, + seeds, + ) + + if dummy_run and skip_attn_for_dummy_run: + # Memory profiling path: block_tables / kv_cache_config are not initialized. + # Since DFlash needs to build its own attention metadata, we must skip the + # preparation in this path and run a minimal forward pass. + self.model.precompute_and_store_context_kv( + self.hidden_states[:num_target_tokens], + self.context_positions[:num_target_tokens], + ) + self._generate_draft( + num_reqs, + num_query_tokens, + attn_metadata=None, + slot_mappings=None, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=CUDAGraphMode.NONE, + ) + return self.draft_tokens[:num_reqs] + + # The query slot mapping is written into the shared BlockTables slot_mappings. + # That buffer's address is what the captured CUDA graph reads from at replay. + assert self.draft_kv_cache_group_id >= 0 + query_slot_mapping = self.block_tables.slot_mappings[ + self.draft_kv_cache_group_id + ] + prepare_dflash_inputs( + self.input_buffers, + query_slot_mapping, + self.context_positions, + self.context_slot_mapping, + self.sample_indices, + self.sample_pos, + self.sample_idx_mapping, + input_batch, + num_sampled, + num_rejected, + last_sampled, + next_prefill_tokens, + self.block_tables.input_block_tables[self.draft_kv_cache_group_id], + self.draft_block_size, + self.parallel_drafting_token_id, + self.num_query_per_req, + self.num_speculative_steps, + self.max_num_reqs, + self.max_num_tokens, + ) + + # Pre-insert context K/V into the cache. Runs eagerly outside the captured graph + # because the context shape varies per step. During dummy runs the block tables + # are placeholders, so we skip the cache write to avoid clobbering real entries. + self.model.precompute_and_store_context_kv( + self.hidden_states[:num_target_tokens], + self.context_positions[:num_target_tokens], + context_slot_mapping=( + None if dummy_run else self.context_slot_mapping[:num_target_tokens] + ), + ) + + # Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs + batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( + self.query_cudagraph_manager, + num_reqs, + num_query_tokens, + uniform_token_count=self.num_query_per_req, + dp_size=self.dp_size, + dp_rank=self.dp_rank, + need_eager=is_profile, + ) + + num_reqs_padded = batch_desc.num_reqs or num_reqs + num_tokens_padded = batch_desc.num_tokens + + # Rebuild the draft attention metadata even when replaying the FULL + # graph so that any attention metadata builder state is updated. + draft_attn_metadata = self._build_draft_attn_metadata( + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + num_tokens_padded=num_tokens_padded, + causal=self.dflash_causal, + ) + draft_slot_mappings_by_layer = build_slot_mappings_by_layer( + self.block_tables.slot_mappings[:, :num_tokens_padded], + self.kv_cache_config, + ) + + if batch_desc.cg_mode == CUDAGraphMode.FULL: + assert self.query_cudagraph_manager is not None + self.query_cudagraph_manager.run_fullgraph(batch_desc) + else: + self._generate_draft( + num_reqs_padded, + num_tokens_padded, + draft_attn_metadata, + draft_slot_mappings_by_layer, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=batch_desc.cg_mode, + ) + + return self.draft_tokens[:num_reqs] + + +@triton.jit +def _prepare_dflash_inputs_kernel( + # Outputs + out_input_ids_ptr, + out_query_positions_ptr, + out_query_start_loc_ptr, + out_seq_lens_ptr, + out_query_slot_mapping_ptr, + out_context_positions_ptr, + out_context_slot_mapping_ptr, + out_sample_indices_ptr, + out_sample_pos_ptr, + out_sample_idx_mapping_ptr, + # Inputs from target batch + target_positions_ptr, + target_query_start_loc_ptr, + idx_mapping_ptr, + last_sampled_ptr, + next_prefill_tokens_ptr, + num_sampled_ptr, + num_rejected_ptr, + # Block table for slot mapping lookup. + block_table_ptr, + block_table_stride, + # Scalars + parallel_drafting_token_id, + block_size, + num_query_per_req, + num_speculative_steps, + max_num_reqs, + max_num_tokens, + PAD_SLOT_ID: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + block_idx = tl.program_id(1) + num_reqs = tl.num_programs(0) + req_state_idx = tl.load(idx_mapping_ptr + req_idx) + + ctx_start = tl.load(target_query_start_loc_ptr + req_idx) + ctx_end = tl.load(target_query_start_loc_ptr + req_idx + 1) + num_ctx = ctx_end - ctx_start + + num_rejected = tl.load(num_rejected_ptr + req_idx) + valid_ctx_end = ctx_end - num_rejected + + num_sampled = tl.load(num_sampled_ptr + req_idx) + if num_sampled > 0: + bonus_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32) + else: + # Chunked prefilling: splice in the next prefill token. + bonus_token = tl.load(next_prefill_tokens_ptr + req_state_idx).to(tl.int32) + + last_valid_pos = tl.load(target_positions_ptr + valid_ctx_end - 1) + query_base = req_idx * num_query_per_req + + j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + is_ctx = j < num_ctx + is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req) + query_off = j - num_ctx + + # --- Context positions / slots --- + ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0) + ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0) + ctx_block_num = ctx_pos // block_size + ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1) + ctx_block_id = tl.load( + block_table_ptr + req_idx * block_table_stride + ctx_block_num, + mask=is_ctx, + other=0, + ).to(tl.int64) + ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size) + tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx) + tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx) + + # --- Query positions / input_ids / slots --- + query_pos = last_valid_pos + 1 + query_off + query_idx = query_base + query_off + is_bonus = is_query & (query_off == 0) + input_id = tl.where(is_bonus, bonus_token, parallel_drafting_token_id) + + q_block_num = query_pos // block_size + q_block_num = tl.minimum(q_block_num, block_table_stride - 1) + q_block_id = tl.load( + block_table_ptr + req_idx * block_table_stride + q_block_num, + mask=is_query, + other=0, + ).to(tl.int64) + q_slot = q_block_id * block_size + (query_pos % block_size) + + tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query) + tl.store(out_query_positions_ptr + query_idx, query_pos, mask=is_query) + tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query) + + # --- Sample indices / positions / idx_mapping (mask tokens only) --- + is_sample = is_query & (query_off > 0) + sample_idx = req_idx * num_speculative_steps + (query_off - 1) + tl.store(out_sample_indices_ptr + sample_idx, query_idx, mask=is_sample) + tl.store(out_sample_pos_ptr + sample_idx, query_pos, mask=is_sample) + tl.store(out_sample_idx_mapping_ptr + sample_idx, req_state_idx, mask=is_sample) + + if block_idx == 0: + tl.store(out_query_start_loc_ptr + req_idx, query_base) + # seq_lens is the absolute sequence length the draft attention + # reads up to (context + query), not just the count of accepted + # tokens this step. + tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req) + if req_idx == num_reqs - 1: + # Pad per-request buffers to max_num_reqs for CUDA graph safety. + last_query_end = num_reqs * num_query_per_req + for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + 1 + tl.store(out_query_start_loc_ptr + block, last_query_end, mask=mask) + for i in range(num_reqs, max_num_reqs, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + tl.store(out_seq_lens_ptr + block, 0, mask=mask) + # Padded sample slots point at query index 0 (a valid row in + # last_hidden_states) so CG replay never reads OOB. + pad_start = num_reqs * num_speculative_steps + pad_end = max_num_reqs * num_speculative_steps + for i in range(pad_start, pad_end, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < pad_end + tl.store(out_sample_indices_ptr + block, 0, mask=mask) + tl.store(out_sample_pos_ptr + block, 0, mask=mask) + tl.store(out_sample_idx_mapping_ptr + block, 0, mask=mask) + # Pad query slot mappings past num_query_tokens with PAD so the + # captured CG sees PAD slots (no K/V write) for replay sizes + # larger than the current request count. + q_pad_start = num_reqs * num_query_per_req + for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_tokens + tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask) + + +def prepare_dflash_inputs( + input_buffers: InputBuffers, + query_slot_mapping: torch.Tensor, + context_positions: torch.Tensor, + context_slot_mapping: torch.Tensor, + sample_indices: torch.Tensor, + sample_pos: torch.Tensor, + sample_idx_mapping: torch.Tensor, + input_batch: InputBatch, + # [num_reqs] + num_sampled: torch.Tensor, + # [num_reqs] + num_rejected: torch.Tensor, + # [max_num_reqs] + last_sampled: torch.Tensor, + # [max_num_reqs] + next_prefill_tokens: torch.Tensor, + # [max_num_reqs, max_num_blocks] + block_table: torch.Tensor, + block_size: int, + parallel_drafting_token_id: int, + num_query_per_req: int, + num_speculative_steps: int, + max_num_reqs: int, + max_num_tokens: int, +) -> None: + num_reqs = input_batch.num_reqs + assert num_reqs > 0 + # Cover the longest possible per-request span (ctx + query). Use the max + # per-request query length, not the total token count across the batch. + max_target_query_len = int(input_batch.num_scheduled_tokens.max()) + max_tokens_per_req = max_target_query_len + num_query_per_req + BLOCK_SIZE = min(256, triton.next_power_of_2(max(1, max_tokens_per_req))) + num_blocks = triton.cdiv(max_tokens_per_req, BLOCK_SIZE) + _prepare_dflash_inputs_kernel[(num_reqs, num_blocks)]( + input_buffers.input_ids, + input_buffers.positions, + input_buffers.query_start_loc, + input_buffers.seq_lens, + query_slot_mapping, + context_positions, + context_slot_mapping, + sample_indices, + sample_pos, + sample_idx_mapping, + input_batch.positions, + input_batch.query_start_loc, + input_batch.idx_mapping, + last_sampled, + next_prefill_tokens, + num_sampled, + num_rejected, + block_table, + block_table.stride(0), + parallel_drafting_token_id, + block_size, + num_query_per_req, + num_speculative_steps, + max_num_reqs, + max_num_tokens, + PAD_SLOT_ID=PAD_SLOT_ID, + BLOCK_SIZE=BLOCK_SIZE, + ) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py new file mode 100644 index 0000000000..f4ea4be8b8 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch.nn as nn + +from vllm.config import ModelConfig, VllmConfig, replace +from vllm.distributed.parallel_state import get_pp_group +from vllm.model_executor.model_loader import get_model +from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share + + +def get_dflash_causal(draft_model_config: ModelConfig) -> bool: + """Whether the DFlash draft uses causal (vs non-causal) attention.""" + dflash_config = getattr(draft_model_config.hf_config, "dflash_config", None) or {} + return dflash_config.get("causal", False) + + +def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module: + from vllm.compilation.backends import set_model_tag + + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + draft_model_config = speculative_config.draft_model_config + # Modify the attention config so that we select an attention backend that matches + # the causal/non-causal mode of the dflash model. + causal = get_dflash_causal(draft_model_config) + draft_vllm_config = replace( + vllm_config, + attention_config=replace( + vllm_config.attention_config, use_non_causal=not causal + ), + ) + with set_model_tag("dflash_head"): + dflash_model = get_model( + vllm_config=draft_vllm_config, model_config=draft_model_config + ) + + target_language_model = ( + target_model.get_language_model() + if hasattr(target_model, "get_language_model") + else target_model + ) + target_inner = target_language_model.model + draft_inner = dflash_model.model + + # Skip embedding sharing under PP — each rank owns its own embedding. + if get_pp_group().world_size == 1: + target_embed = getattr(target_inner, "embed_tokens", None) or getattr( + target_inner, "embedding", None + ) + draft_embed = getattr(draft_inner, "embed_tokens", None) + if target_embed is not None and _should_share( + dflash_model, "has_own_embed_tokens", draft_embed, target_embed + ): + if draft_embed is not None: + del draft_inner.embed_tokens + draft_inner.embed_tokens = target_embed + + # Share lm_head with the target unless the draft remaps vocab via + # draft_id_to_target_id (in which case its own lm_head is required). + target_lm_head = getattr(target_model, "lm_head", None) + draft_lm_head = getattr(dflash_model, "lm_head", None) + if ( + target_lm_head is not None + and draft_lm_head is not None + and getattr(dflash_model, "draft_id_to_target_id", None) is None + ): + del dflash_model.lm_head + dflash_model.lm_head = target_lm_head + + return dflash_model diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py index d805c88582..360f64921e 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py @@ -38,9 +38,12 @@ def get_eagle3_aux_layers_from_config( if not (spec_config and spec_config.draft_model_config): return None hf_config = spec_config.draft_model_config.hf_config - if not hasattr(hf_config, "eagle_aux_hidden_state_layer_ids"): - return None - layer_ids = hf_config.eagle_aux_hidden_state_layer_ids + layer_ids = getattr(hf_config, "eagle_aux_hidden_state_layer_ids", None) + if not layer_ids: + dflash_config = getattr(hf_config, "dflash_config", None) + if dflash_config and isinstance(dflash_config, dict): + # Add 1 to convert DFlash's aux layer id semantics + layer_ids = [i + 1 for i in (dflash_config.get("target_layer_ids") or [])] if layer_ids and isinstance(layer_ids, (list, tuple)): return tuple(layer_ids) return None diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index e8fa8af53b..4fd7cce36b 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -21,6 +21,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import ( ) from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample class BaseSpeculator(ABC): @@ -178,9 +179,15 @@ class DraftModelSpeculator(BaseSpeculator): num_reqs: int, num_reqs_padded: int, num_tokens_padded: int, + num_query_per_req: int = 1, + causal: bool = True, ) -> dict[str, Any] | None: - query_start_loc_cpu = torch.clamp( - self.arange[: num_reqs_padded + 1], max=num_reqs + # Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req. + # Clamp keeps the series non-decreasing past num_reqs, which some + # attention backends require. + query_start_loc_cpu = ( + torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs) + * num_query_per_req ) block_tables = [ x[:num_reqs_padded] for x in self.block_tables.input_block_tables @@ -194,15 +201,44 @@ class DraftModelSpeculator(BaseSpeculator): : num_reqs_padded + 1 ], query_start_loc_cpu=query_start_loc_cpu, - max_query_len=1, + max_query_len=num_query_per_req, seq_lens=self.input_buffers.seq_lens[:num_reqs_padded], max_seq_len=self.draft_max_seq_len, block_tables=block_tables, slot_mappings=slot_mappings, kv_cache_config=self.kv_cache_config, + causal=causal, ) return attn_metadata + def sample_draft( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + idx_mapping: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, + draft_step: torch.Tensor, + draft_logits: torch.Tensor | None, + ) -> torch.Tensor: + logits = self.model.compute_logits(hidden_states) + if draft_logits is not None: + # NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise + # used for draft and target sampling. + return gumbel_sample( + logits, + idx_mapping, + temperature, + seeds, + positions + 1, + apply_temperature=True, + output_processed_logits=draft_logits, + output_processed_logits_col=draft_step, + use_fp64=self.use_fp64_gumbel, + ) + else: + return logits.argmax(dim=-1) + def _copy_request_inputs( self, num_reqs: int, diff --git a/vllm/v1/worker/gpu/spec_decode/utils.py b/vllm/v1/worker/gpu/spec_decode/utils.py index e1fa21aeb8..7bfd981ee0 100644 --- a/vllm/v1/worker/gpu/spec_decode/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/utils.py @@ -45,3 +45,22 @@ class DraftTokensHandler: # This case only happens when async scheduling is disabled. draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids] return DraftTokenIds(self.req_ids, draft_token_ids) + + +def get_parallel_drafting_token_id(hf_config) -> int: + """Resolve the mask token id used for parallel drafting slots. + + Checks (in order): `dflash_config.mask_token_id`, `pard_token`, + `ptd_token_id`. Raises ValueError if none are present. + """ + dflash_config = getattr(hf_config, "dflash_config", None) or {} + if "mask_token_id" in dflash_config: + return int(dflash_config["mask_token_id"]) + if hasattr(hf_config, "pard_token"): + return int(hf_config.pard_token) + if hasattr(hf_config, "ptd_token_id"): + return int(hf_config.ptd_token_id) + raise ValueError( + "Model config must specify `dflash_config.mask_token_id`," + " `pard_token`, or `ptd_token_id` for parallel drafting." + )