[MRV2][Spec Decode] DFlash (#44586)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Benjamin Chislett
2026-06-10 08:47:46 -07:00
committed by GitHub
co-authored by Giancarlo Delfin
parent 4673ca1d78
commit 0bae1d3848
14 changed files with 868 additions and 47 deletions
+12 -2
View File
@@ -1300,13 +1300,18 @@ def dflash_config():
)
def test_dflash_acceptance_rates(dflash_config):
@pytest.mark.parametrize("use_mrv2", [False, True])
def test_dflash_acceptance_rates(
monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config
):
"""
E2E test for DFlash (block diffusion) speculative decoding.
Runs acceptance rate validation on GSM8k, MT-Bench, and HumanEval
comparing against baseline results from the paper (Table 1).
See https://github.com/z-lab/dflash/blob/main/benchmark_sglang.py for methodology.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
spec_llm = LLM(**dflash_config)
max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k
@@ -1414,11 +1419,16 @@ def test_synthetic_acceptance_rate():
cleanup_dist_env_and_memory()
def test_dflash_correctness(dflash_config):
@pytest.mark.parametrize("use_mrv2", [False, True])
def test_dflash_correctness(
monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config
):
"""
E2E test for DFlash (block diffusion) speculative decoding.
Ensures output correctness on GSM8k, with cudagraphs and batching on.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
spec_llm = LLM(**dflash_config)
# Evaluate GSM8k accuracy (Qwen3-8B ref: ~87-92% on GSM8k)
+8 -4
View File
@@ -2008,12 +2008,16 @@ class VllmConfig:
# TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
if speculative_config.method in ("ngram", "ngram_gpu"):
unsupported.append("ngram/ngram_gpu speculative decoding")
elif speculative_config.method not in ("eagle", "eagle3", "mtp"):
elif speculative_config.method not in ("eagle", "eagle3", "mtp", "dflash"):
unsupported.append(f"speculative method '{speculative_config.method}'")
# V2 EagleSpeculator does not support parallel_drafting (required by PEagle)
if speculative_config.parallel_drafting:
unsupported.append("parallel drafting for speculative decoding")
# V2 EagleSpeculator does not support parallel_drafting (for P-Eagle)
# DFlash uses parallel drafting natively in V2 via DFlashSpeculator.
if (
speculative_config.parallel_drafting
and speculative_config.method != "dflash"
):
unsupported.append("parallel drafting for EAGLE speculative decoding")
if (
speculative_config.method == "eagle3"
+2 -1
View File
@@ -394,6 +394,7 @@ def build_attn_metadata(
positions: torch.Tensor | None = None,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
for_cudagraph_capture: bool = False,
causal: bool = True,
) -> dict[str, Any]:
seq_lens = seq_lens[:num_reqs]
if dcp_local_seq_lens is not None:
@@ -423,7 +424,7 @@ def build_attn_metadata(
max_query_len=max_query_len,
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=True,
causal=causal,
dcp_local_seq_lens=dcp_local_seq_lens,
positions=positions,
**common_attn_metadata_extra_kwargs,
+6 -3
View File
@@ -193,11 +193,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.is_last_pp_rank:
self.speculator = init_speculator(self.vllm_config, self.device)
if self.speculative_config.method == "eagle3":
# EAGLE3 may require auxiliary hidden states from target model outputs.
if self.speculative_config.method in ("eagle3", "dflash"):
# Drafting may require auxiliary hidden states from target model outputs
self.use_aux_hidden_state_outputs = True
if self.use_pp:
raise ValueError("EAGLE3 with pipeline parallel is not supported.")
raise ValueError(
f"{self.speculative_config.method} with pipeline parallel "
"is not supported."
)
# Draft tokens propagation - for spec-dec + struct outputs.
self.draft_tokens_handler = DraftTokensHandler(self.device)
+12 -1
View File
@@ -91,6 +91,7 @@ def gumbel_block_argmax(
vocab_size,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
PER_TOKEN_COL: tl.constexpr = False,
):
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
@@ -103,7 +104,10 @@ def gumbel_block_argmax(
if processed_logits_ptr is not None:
# Store the temperature-applied logits.
if processed_logits_col_ptr is not None:
col = tl.load(processed_logits_col_ptr)
if PER_TOKEN_COL:
col = tl.load(processed_logits_col_ptr + token_idx)
else:
col = tl.load(processed_logits_col_ptr)
else:
col = 0
tl.store(
@@ -158,6 +162,7 @@ def _gumbel_sample_kernel(
BLOCK_SIZE: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
PER_TOKEN_COL: tl.constexpr,
):
token_idx = tl.program_id(0)
block_idx = tl.program_id(1)
@@ -185,6 +190,7 @@ def _gumbel_sample_kernel(
vocab_size,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
USE_FP64=USE_FP64,
PER_TOKEN_COL=PER_TOKEN_COL,
)
token_id = block_idx * BLOCK_SIZE + idx
tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id)
@@ -208,6 +214,10 @@ def gumbel_sample(
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
local_max_dtype = torch.float64 if use_fp64 else torch.float32
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
per_token_col = (
output_processed_logits_col is not None
and output_processed_logits_col.dim() > 0
)
_gumbel_sample_kernel[(num_tokens, num_blocks)](
local_argmax,
local_argmax.stride(0),
@@ -226,6 +236,7 @@ def gumbel_sample(
BLOCK_SIZE=BLOCK_SIZE,
APPLY_TEMPERATURE=apply_temperature,
USE_FP64=use_fp64,
PER_TOKEN_COL=per_token_col,
)
# NOTE(woosuk): Use int64 for later indexing.
max_block_idx = local_max.argmax(dim=-1, keepdim=True)
+7 -1
View File
@@ -8,7 +8,13 @@ from vllm.config import VllmConfig
def init_speculator(vllm_config: VllmConfig, device: torch.device):
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
if speculative_config.use_gemma4_mtp():
if speculative_config.method == "dflash":
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
DFlashSpeculator,
)
return DFlashSpeculator(vllm_config, device)
elif speculative_config.use_gemma4_mtp():
from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import (
Gemma4Speculator,
)
@@ -18,7 +18,6 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
DecodeSpeculatorCudaGraphManager,
PrefillSpeculatorCudaGraphManager,
@@ -279,34 +278,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
return self.draft_tokens[:num_reqs]
def sample_draft(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
) -> torch.Tensor:
logits = self.model.compute_logits(hidden_states)
if draft_logits is not None:
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
# used for draft and target sampling.
return gumbel_sample(
logits,
idx_mapping,
temperature,
seeds,
positions + 1,
apply_temperature=True,
output_processed_logits=draft_logits,
output_processed_logits_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
else:
return logits.argmax(dim=-1)
@torch.inference_mode()
def _run_model(
self,
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
build_slot_mappings_by_layer,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionState,
BatchExecutionDescriptor,
CudaGraphManager,
)
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.utils import AttentionGroup
def _prepare_dflash_inputs_to_capture(
num_reqs: int,
num_tokens: int,
input_buffers: InputBuffers,
block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
max_model_len: int,
skip_attn: bool,
causal: bool,
) -> AttentionState:
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens)
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, kv_cache_config
)
attn_metadata = None
if not skip_attn:
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
attn_metadata = build_attn_metadata(
attn_groups=attn_groups,
num_reqs=num_reqs,
num_tokens=num_tokens,
query_start_loc_gpu=input_batch.query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=num_tokens // num_reqs,
seq_lens=input_batch.seq_lens,
max_seq_len=max_model_len,
block_tables=input_block_tables,
slot_mappings=slot_mappings,
kv_cache_config=kv_cache_config,
for_cudagraph_capture=True,
causal=causal,
)
return AttentionState(attn_metadata, slot_mappings_by_layer)
class DFlashCudaGraphManager(CudaGraphManager):
"""DFlash CudaGraphManager for the parallel-drafting query forward,
building its own attention metadata from scratch."""
def __init__(self, *args, causal: bool = False, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.causal = causal
def capture(
self,
forward_fn: Callable,
input_buffers: InputBuffers,
block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
max_model_len: int,
progress_bar_desc: str = "Capturing CUDA graphs",
) -> None:
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
num_tokens_across_dp = (
torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu")
if self.dp_size > 1
else None
)
attn_state = _prepare_dflash_inputs_to_capture(
num_reqs,
num_tokens,
input_buffers,
block_tables,
attn_groups,
kv_cache_config,
max_model_len,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
causal=self.causal,
)
attn_metadata, slot_mappings = attn_state
fwd = lambda cg_mode: forward_fn(
num_reqs,
num_tokens,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cg_mode,
)
return fwd, attn_state
super().capture(create_forward_fn, progress_bar_desc)
@@ -0,0 +1,573 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
get_dflash_causal,
load_dflash_model,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
logger = init_logger(__name__)
class DFlashSpeculator(DraftModelSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
self.hidden_states = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
# Multimodal inputs not currently supported.
self.supports_mm_inputs = False
# Each request emits exactly (bonus + N mask) query tokens per step.
self.num_query_per_req = 1 + self.num_speculative_steps
self.parallel_drafting_token_id = get_parallel_drafting_token_id(
self.draft_model_config.hf_config
)
self.dflash_causal = get_dflash_causal(self.draft_model_config)
# Buffers for context K/V precomputation. Populated by prepare_dflash_inputs,
# and processed by the model's precompute_and_store_context_kv method.
# NOT captured by CUDA graphs.
self.context_positions = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=device
)
self.context_slot_mapping = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=device
)
# Per-mask-token sampling buffers. Flattened from (num_reqs, num_spec_tokens).
max_num_sampled_tokens = self.max_num_reqs * self.num_speculative_steps
self.sample_indices = torch.zeros(
max_num_sampled_tokens, dtype=torch.int64, device=device
)
self.sample_pos = torch.zeros(
max_num_sampled_tokens, dtype=torch.int64, device=device
)
self.sample_idx_mapping = torch.zeros(
max_num_sampled_tokens, dtype=torch.int32, device=device
)
# [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into
# draft_logits[req, step, :].
self.sample_col = torch.arange(
self.num_speculative_steps, dtype=torch.int32, device=device
).repeat(self.max_num_reqs)
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
self.draft_kv_cache_group_id: int = -1
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# PIECEWISE cudagraphs are not supported for dflash
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
else:
cudagraph_mode = CUDAGraphMode.NONE
self.query_cudagraph_manager = DFlashCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
decode_query_len=self.num_query_per_req,
causal=self.dflash_causal,
)
def capture(self, attn_states: dict | None = None) -> None:
logger.info("Capturing model for DFlash speculator...")
# Reset sampling indices to zero to prevent stale values from prior
# dummy runs from being baked into the captured graph.
self.sample_indices.zero_()
self.sample_pos.zero_()
self.sample_idx_mapping.zero_()
assert self.query_cudagraph_manager is not None
self.query_cudagraph_manager.capture(
self._generate_draft,
self.input_buffers,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
self.max_model_len,
progress_bar_desc="Capturing dflash CUDA graphs",
)
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
return load_dflash_model(target_model, self.vllm_config)
def set_attn(
self,
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
) -> None:
super().set_attn(model_state, kv_cache_config, block_tables)
# DFlash precomputes context K/V with a single block_size; mixing
# kv-cache groups would silently corrupt the cache for the non-matching group.
draft_groups = [gid for gid, g in enumerate(self.attn_groups) if g]
assert len(draft_groups) == 1, (
"DFlash currently requires all draft attention layers to share "
"a single kv-cache group."
)
self.draft_kv_cache_group_id = draft_groups[0]
self.draft_block_size = self.block_tables.block_sizes[
self.draft_kv_cache_group_id
]
@torch.inference_mode()
def _run_model(
self,
num_tokens: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> torch.Tensor:
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
with set_forward_context(
attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
cudagraph_runtime_mode=cudagraph_runtime_mode,
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
):
last_hidden_states = self.model(
input_ids=self.input_buffers.input_ids[:num_tokens],
positions=self.input_buffers.positions[:num_tokens],
inputs_embeds=None,
)
return last_hidden_states
def _generate_draft(
self,
num_reqs: int,
num_tokens_padded: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> None:
last_hidden_states = self._run_model(
num_tokens_padded,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
)
num_sample = num_reqs * self.num_speculative_steps
sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]]
draft_tokens = self.sample_draft(
sample_hidden_states,
self.sample_pos[:num_sample],
self.sample_idx_mapping[:num_sample],
self.temperature,
self.seeds,
self.sample_col[:num_sample],
self.draft_logits,
)
self.draft_tokens[:num_reqs] = draft_tokens.view(
num_reqs, self.num_speculative_steps
)
def _build_draft_attn_metadata(
self,
num_reqs: int,
num_reqs_padded: int,
num_tokens_padded: int,
num_query_per_req: int | None = None,
causal: bool = False,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
assert num_query_per_req is None # Omitted for DFlash, read from self instead
return super()._build_draft_attn_metadata(
num_reqs,
num_reqs_padded,
num_tokens_padded,
num_query_per_req=self.num_query_per_req,
causal=causal,
)
@torch.inference_mode()
def propose(
self,
input_batch: InputBatch,
attn_metadata: dict[str, Any],
slot_mappings: dict[str, torch.Tensor],
# [num_tokens, hidden_size]
last_hidden_states: torch.Tensor,
# num_layers x [num_tokens, hidden_size]
aux_hidden_states: list[torch.Tensor] | None,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_target_tokens = input_batch.num_tokens
num_query_tokens = num_reqs * self.num_query_per_req
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
self.draft_max_seq_len = min(
max_seq_len + self.num_query_per_req, self.max_model_len
)
# NOTE: To avoid CPU-GPU synchronization without CPU knowing the
# number of rejected tokens, we maintain the size of input_ids and
# hidden_states the same as the target model's. This means, we pad each
# request's query length to include any rejected positions.
if aux_hidden_states:
hidden_states = self.model.combine_hidden_states(
torch.cat(aux_hidden_states, dim=-1)
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens])
self._copy_request_inputs(
num_reqs,
input_batch.idx_mapping,
temperature,
seeds,
)
if dummy_run and skip_attn_for_dummy_run:
# Memory profiling path: block_tables / kv_cache_config are not initialized.
# Since DFlash needs to build its own attention metadata, we must skip the
# preparation in this path and run a minimal forward pass.
self.model.precompute_and_store_context_kv(
self.hidden_states[:num_target_tokens],
self.context_positions[:num_target_tokens],
)
self._generate_draft(
num_reqs,
num_query_tokens,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
return self.draft_tokens[:num_reqs]
# The query slot mapping is written into the shared BlockTables slot_mappings.
# That buffer's address is what the captured CUDA graph reads from at replay.
assert self.draft_kv_cache_group_id >= 0
query_slot_mapping = self.block_tables.slot_mappings[
self.draft_kv_cache_group_id
]
prepare_dflash_inputs(
self.input_buffers,
query_slot_mapping,
self.context_positions,
self.context_slot_mapping,
self.sample_indices,
self.sample_pos,
self.sample_idx_mapping,
input_batch,
num_sampled,
num_rejected,
last_sampled,
next_prefill_tokens,
self.block_tables.input_block_tables[self.draft_kv_cache_group_id],
self.draft_block_size,
self.parallel_drafting_token_id,
self.num_query_per_req,
self.num_speculative_steps,
self.max_num_reqs,
self.max_num_tokens,
)
# Pre-insert context K/V into the cache. Runs eagerly outside the captured graph
# because the context shape varies per step. During dummy runs the block tables
# are placeholders, so we skip the cache write to avoid clobbering real entries.
self.model.precompute_and_store_context_kv(
self.hidden_states[:num_target_tokens],
self.context_positions[:num_target_tokens],
context_slot_mapping=(
None if dummy_run else self.context_slot_mapping[:num_target_tokens]
),
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.query_cudagraph_manager,
num_reqs,
num_query_tokens,
uniform_token_count=self.num_query_per_req,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
num_reqs_padded = batch_desc.num_reqs or num_reqs
num_tokens_padded = batch_desc.num_tokens
# Rebuild the draft attention metadata even when replaying the FULL
# graph so that any attention metadata builder state is updated.
draft_attn_metadata = self._build_draft_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
num_tokens_padded=num_tokens_padded,
causal=self.dflash_causal,
)
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
self.block_tables.slot_mappings[:, :num_tokens_padded],
self.kv_cache_config,
)
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.query_cudagraph_manager is not None
self.query_cudagraph_manager.run_fullgraph(batch_desc)
else:
self._generate_draft(
num_reqs_padded,
num_tokens_padded,
draft_attn_metadata,
draft_slot_mappings_by_layer,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=batch_desc.cg_mode,
)
return self.draft_tokens[:num_reqs]
@triton.jit
def _prepare_dflash_inputs_kernel(
# Outputs
out_input_ids_ptr,
out_query_positions_ptr,
out_query_start_loc_ptr,
out_seq_lens_ptr,
out_query_slot_mapping_ptr,
out_context_positions_ptr,
out_context_slot_mapping_ptr,
out_sample_indices_ptr,
out_sample_pos_ptr,
out_sample_idx_mapping_ptr,
# Inputs from target batch
target_positions_ptr,
target_query_start_loc_ptr,
idx_mapping_ptr,
last_sampled_ptr,
next_prefill_tokens_ptr,
num_sampled_ptr,
num_rejected_ptr,
# Block table for slot mapping lookup.
block_table_ptr,
block_table_stride,
# Scalars
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
PAD_SLOT_ID: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
block_idx = tl.program_id(1)
num_reqs = tl.num_programs(0)
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
ctx_start = tl.load(target_query_start_loc_ptr + req_idx)
ctx_end = tl.load(target_query_start_loc_ptr + req_idx + 1)
num_ctx = ctx_end - ctx_start
num_rejected = tl.load(num_rejected_ptr + req_idx)
valid_ctx_end = ctx_end - num_rejected
num_sampled = tl.load(num_sampled_ptr + req_idx)
if num_sampled > 0:
bonus_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32)
else:
# Chunked prefilling: splice in the next prefill token.
bonus_token = tl.load(next_prefill_tokens_ptr + req_state_idx).to(tl.int32)
last_valid_pos = tl.load(target_positions_ptr + valid_ctx_end - 1)
query_base = req_idx * num_query_per_req
j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
is_ctx = j < num_ctx
is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req)
query_off = j - num_ctx
# --- Context positions / slots ---
ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0)
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0)
ctx_block_num = ctx_pos // block_size
ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1)
ctx_block_id = tl.load(
block_table_ptr + req_idx * block_table_stride + ctx_block_num,
mask=is_ctx,
other=0,
).to(tl.int64)
ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size)
tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx)
tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx)
# --- Query positions / input_ids / slots ---
query_pos = last_valid_pos + 1 + query_off
query_idx = query_base + query_off
is_bonus = is_query & (query_off == 0)
input_id = tl.where(is_bonus, bonus_token, parallel_drafting_token_id)
q_block_num = query_pos // block_size
q_block_num = tl.minimum(q_block_num, block_table_stride - 1)
q_block_id = tl.load(
block_table_ptr + req_idx * block_table_stride + q_block_num,
mask=is_query,
other=0,
).to(tl.int64)
q_slot = q_block_id * block_size + (query_pos % block_size)
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
tl.store(out_query_positions_ptr + query_idx, query_pos, mask=is_query)
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
# --- Sample indices / positions / idx_mapping (mask tokens only) ---
is_sample = is_query & (query_off > 0)
sample_idx = req_idx * num_speculative_steps + (query_off - 1)
tl.store(out_sample_indices_ptr + sample_idx, query_idx, mask=is_sample)
tl.store(out_sample_pos_ptr + sample_idx, query_pos, mask=is_sample)
tl.store(out_sample_idx_mapping_ptr + sample_idx, req_state_idx, mask=is_sample)
if block_idx == 0:
tl.store(out_query_start_loc_ptr + req_idx, query_base)
# seq_lens is the absolute sequence length the draft attention
# reads up to (context + query), not just the count of accepted
# tokens this step.
tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req)
if req_idx == num_reqs - 1:
# Pad per-request buffers to max_num_reqs for CUDA graph safety.
last_query_end = num_reqs * num_query_per_req
for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs + 1
tl.store(out_query_start_loc_ptr + block, last_query_end, mask=mask)
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_reqs
tl.store(out_seq_lens_ptr + block, 0, mask=mask)
# Padded sample slots point at query index 0 (a valid row in
# last_hidden_states) so CG replay never reads OOB.
pad_start = num_reqs * num_speculative_steps
pad_end = max_num_reqs * num_speculative_steps
for i in range(pad_start, pad_end, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < pad_end
tl.store(out_sample_indices_ptr + block, 0, mask=mask)
tl.store(out_sample_pos_ptr + block, 0, mask=mask)
tl.store(out_sample_idx_mapping_ptr + block, 0, mask=mask)
# Pad query slot mappings past num_query_tokens with PAD so the
# captured CG sees PAD slots (no K/V write) for replay sizes
# larger than the current request count.
q_pad_start = num_reqs * num_query_per_req
for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_tokens
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
def prepare_dflash_inputs(
input_buffers: InputBuffers,
query_slot_mapping: torch.Tensor,
context_positions: torch.Tensor,
context_slot_mapping: torch.Tensor,
sample_indices: torch.Tensor,
sample_pos: torch.Tensor,
sample_idx_mapping: torch.Tensor,
input_batch: InputBatch,
# [num_reqs]
num_sampled: torch.Tensor,
# [num_reqs]
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs, max_num_blocks]
block_table: torch.Tensor,
block_size: int,
parallel_drafting_token_id: int,
num_query_per_req: int,
num_speculative_steps: int,
max_num_reqs: int,
max_num_tokens: int,
) -> None:
num_reqs = input_batch.num_reqs
assert num_reqs > 0
# Cover the longest possible per-request span (ctx + query). Use the max
# per-request query length, not the total token count across the batch.
max_target_query_len = int(input_batch.num_scheduled_tokens.max())
max_tokens_per_req = max_target_query_len + num_query_per_req
BLOCK_SIZE = min(256, triton.next_power_of_2(max(1, max_tokens_per_req)))
num_blocks = triton.cdiv(max_tokens_per_req, BLOCK_SIZE)
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
input_buffers.input_ids,
input_buffers.positions,
input_buffers.query_start_loc,
input_buffers.seq_lens,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
input_batch.positions,
input_batch.query_start_loc,
input_batch.idx_mapping,
last_sampled,
next_prefill_tokens,
num_sampled,
num_rejected,
block_table,
block_table.stride(0),
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
PAD_SLOT_ID=PAD_SLOT_ID,
BLOCK_SIZE=BLOCK_SIZE,
)
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
"""Whether the DFlash draft uses causal (vs non-causal) attention."""
dflash_config = getattr(draft_model_config.hf_config, "dflash_config", None) or {}
return dflash_config.get("causal", False)
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
from vllm.compilation.backends import set_model_tag
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
# Modify the attention config so that we select an attention backend that matches
# the causal/non-causal mode of the dflash model.
causal = get_dflash_causal(draft_model_config)
draft_vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config, use_non_causal=not causal
),
)
with set_model_tag("dflash_head"):
dflash_model = get_model(
vllm_config=draft_vllm_config, model_config=draft_model_config
)
target_language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
target_inner = target_language_model.model
draft_inner = dflash_model.model
# Skip embedding sharing under PP — each rank owns its own embedding.
if get_pp_group().world_size == 1:
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
dflash_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
# Share lm_head with the target unless the draft remaps vocab via
# draft_id_to_target_id (in which case its own lm_head is required).
target_lm_head = getattr(target_model, "lm_head", None)
draft_lm_head = getattr(dflash_model, "lm_head", None)
if (
target_lm_head is not None
and draft_lm_head is not None
and getattr(dflash_model, "draft_id_to_target_id", None) is None
):
del dflash_model.lm_head
dflash_model.lm_head = target_lm_head
return dflash_model
@@ -38,9 +38,12 @@ def get_eagle3_aux_layers_from_config(
if not (spec_config and spec_config.draft_model_config):
return None
hf_config = spec_config.draft_model_config.hf_config
if not hasattr(hf_config, "eagle_aux_hidden_state_layer_ids"):
return None
layer_ids = hf_config.eagle_aux_hidden_state_layer_ids
layer_ids = getattr(hf_config, "eagle_aux_hidden_state_layer_ids", None)
if not layer_ids:
dflash_config = getattr(hf_config, "dflash_config", None)
if dflash_config and isinstance(dflash_config, dict):
# Add 1 to convert DFlash's aux layer id semantics
layer_ids = [i + 1 for i in (dflash_config.get("target_layer_ids") or [])]
if layer_ids and isinstance(layer_ids, (list, tuple)):
return tuple(layer_ids)
return None
+39 -3
View File
@@ -21,6 +21,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
)
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
class BaseSpeculator(ABC):
@@ -178,9 +179,15 @@ class DraftModelSpeculator(BaseSpeculator):
num_reqs: int,
num_reqs_padded: int,
num_tokens_padded: int,
num_query_per_req: int = 1,
causal: bool = True,
) -> dict[str, Any] | None:
query_start_loc_cpu = torch.clamp(
self.arange[: num_reqs_padded + 1], max=num_reqs
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some
# attention backends require.
query_start_loc_cpu = (
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
* num_query_per_req
)
block_tables = [
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
@@ -194,15 +201,44 @@ class DraftModelSpeculator(BaseSpeculator):
: num_reqs_padded + 1
],
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=1,
max_query_len=num_query_per_req,
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
max_seq_len=self.draft_max_seq_len,
block_tables=block_tables,
slot_mappings=slot_mappings,
kv_cache_config=self.kv_cache_config,
causal=causal,
)
return attn_metadata
def sample_draft(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
) -> torch.Tensor:
logits = self.model.compute_logits(hidden_states)
if draft_logits is not None:
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
# used for draft and target sampling.
return gumbel_sample(
logits,
idx_mapping,
temperature,
seeds,
positions + 1,
apply_temperature=True,
output_processed_logits=draft_logits,
output_processed_logits_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
else:
return logits.argmax(dim=-1)
def _copy_request_inputs(
self,
num_reqs: int,
+19
View File
@@ -45,3 +45,22 @@ class DraftTokensHandler:
# This case only happens when async scheduling is disabled.
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
return DraftTokenIds(self.req_ids, draft_token_ids)
def get_parallel_drafting_token_id(hf_config) -> int:
"""Resolve the mask token id used for parallel drafting slots.
Checks (in order): `dflash_config.mask_token_id`, `pard_token`,
`ptd_token_id`. Raises ValueError if none are present.
"""
dflash_config = getattr(hf_config, "dflash_config", None) or {}
if "mask_token_id" in dflash_config:
return int(dflash_config["mask_token_id"])
if hasattr(hf_config, "pard_token"):
return int(hf_config.pard_token)
if hasattr(hf_config, "ptd_token_id"):
return int(hf_config.ptd_token_id)
raise ValueError(
"Model config must specify `dflash_config.mask_token_id`,"
" `pard_token`, or `ptd_token_id` for parallel drafting."
)