[MRV2][Spec Decode] DFlash (#44586)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
co-authored by
Giancarlo Delfin
parent
4673ca1d78
commit
0bae1d3848
@@ -1300,13 +1300,18 @@ def dflash_config():
|
||||
)
|
||||
|
||||
|
||||
def test_dflash_acceptance_rates(dflash_config):
|
||||
@pytest.mark.parametrize("use_mrv2", [False, True])
|
||||
def test_dflash_acceptance_rates(
|
||||
monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config
|
||||
):
|
||||
"""
|
||||
E2E test for DFlash (block diffusion) speculative decoding.
|
||||
Runs acceptance rate validation on GSM8k, MT-Bench, and HumanEval
|
||||
comparing against baseline results from the paper (Table 1).
|
||||
See https://github.com/z-lab/dflash/blob/main/benchmark_sglang.py for methodology.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
|
||||
|
||||
spec_llm = LLM(**dflash_config)
|
||||
|
||||
max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k
|
||||
@@ -1414,11 +1419,16 @@ def test_synthetic_acceptance_rate():
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def test_dflash_correctness(dflash_config):
|
||||
@pytest.mark.parametrize("use_mrv2", [False, True])
|
||||
def test_dflash_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch, use_mrv2: bool, dflash_config
|
||||
):
|
||||
"""
|
||||
E2E test for DFlash (block diffusion) speculative decoding.
|
||||
Ensures output correctness on GSM8k, with cudagraphs and batching on.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
|
||||
|
||||
spec_llm = LLM(**dflash_config)
|
||||
|
||||
# Evaluate GSM8k accuracy (Qwen3-8B ref: ~87-92% on GSM8k)
|
||||
|
||||
+8
-4
@@ -2008,12 +2008,16 @@ class VllmConfig:
|
||||
# TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
|
||||
if speculative_config.method in ("ngram", "ngram_gpu"):
|
||||
unsupported.append("ngram/ngram_gpu speculative decoding")
|
||||
elif speculative_config.method not in ("eagle", "eagle3", "mtp"):
|
||||
elif speculative_config.method not in ("eagle", "eagle3", "mtp", "dflash"):
|
||||
unsupported.append(f"speculative method '{speculative_config.method}'")
|
||||
|
||||
# V2 EagleSpeculator does not support parallel_drafting (required by PEagle)
|
||||
if speculative_config.parallel_drafting:
|
||||
unsupported.append("parallel drafting for speculative decoding")
|
||||
# V2 EagleSpeculator does not support parallel_drafting (for P-Eagle)
|
||||
# DFlash uses parallel drafting natively in V2 via DFlashSpeculator.
|
||||
if (
|
||||
speculative_config.parallel_drafting
|
||||
and speculative_config.method != "dflash"
|
||||
):
|
||||
unsupported.append("parallel drafting for EAGLE speculative decoding")
|
||||
|
||||
if (
|
||||
speculative_config.method == "eagle3"
|
||||
|
||||
@@ -394,6 +394,7 @@ def build_attn_metadata(
|
||||
positions: torch.Tensor | None = None,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
for_cudagraph_capture: bool = False,
|
||||
causal: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
if dcp_local_seq_lens is not None:
|
||||
@@ -423,7 +424,7 @@ def build_attn_metadata(
|
||||
max_query_len=max_query_len,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=True,
|
||||
causal=causal,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
positions=positions,
|
||||
**common_attn_metadata_extra_kwargs,
|
||||
|
||||
@@ -193,11 +193,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.is_last_pp_rank:
|
||||
self.speculator = init_speculator(self.vllm_config, self.device)
|
||||
|
||||
if self.speculative_config.method == "eagle3":
|
||||
# EAGLE3 may require auxiliary hidden states from target model outputs.
|
||||
if self.speculative_config.method in ("eagle3", "dflash"):
|
||||
# Drafting may require auxiliary hidden states from target model outputs
|
||||
self.use_aux_hidden_state_outputs = True
|
||||
if self.use_pp:
|
||||
raise ValueError("EAGLE3 with pipeline parallel is not supported.")
|
||||
raise ValueError(
|
||||
f"{self.speculative_config.method} with pipeline parallel "
|
||||
"is not supported."
|
||||
)
|
||||
|
||||
# Draft tokens propagation - for spec-dec + struct outputs.
|
||||
self.draft_tokens_handler = DraftTokensHandler(self.device)
|
||||
|
||||
@@ -91,6 +91,7 @@ def gumbel_block_argmax(
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr = False,
|
||||
):
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
@@ -103,7 +104,10 @@ def gumbel_block_argmax(
|
||||
if processed_logits_ptr is not None:
|
||||
# Store the temperature-applied logits.
|
||||
if processed_logits_col_ptr is not None:
|
||||
col = tl.load(processed_logits_col_ptr)
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(processed_logits_col_ptr + token_idx)
|
||||
else:
|
||||
col = tl.load(processed_logits_col_ptr)
|
||||
else:
|
||||
col = 0
|
||||
tl.store(
|
||||
@@ -158,6 +162,7 @@ def _gumbel_sample_kernel(
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
@@ -185,6 +190,7 @@ def _gumbel_sample_kernel(
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
USE_FP64=USE_FP64,
|
||||
PER_TOKEN_COL=PER_TOKEN_COL,
|
||||
)
|
||||
token_id = block_idx * BLOCK_SIZE + idx
|
||||
tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id)
|
||||
@@ -208,6 +214,10 @@ def gumbel_sample(
|
||||
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
|
||||
local_max_dtype = torch.float64 if use_fp64 else torch.float32
|
||||
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
|
||||
per_token_col = (
|
||||
output_processed_logits_col is not None
|
||||
and output_processed_logits_col.dim() > 0
|
||||
)
|
||||
_gumbel_sample_kernel[(num_tokens, num_blocks)](
|
||||
local_argmax,
|
||||
local_argmax.stride(0),
|
||||
@@ -226,6 +236,7 @@ def gumbel_sample(
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
APPLY_TEMPERATURE=apply_temperature,
|
||||
USE_FP64=use_fp64,
|
||||
PER_TOKEN_COL=per_token_col,
|
||||
)
|
||||
# NOTE(woosuk): Use int64 for later indexing.
|
||||
max_block_idx = local_max.argmax(dim=-1, keepdim=True)
|
||||
|
||||
@@ -8,7 +8,13 @@ from vllm.config import VllmConfig
|
||||
def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
if speculative_config.use_gemma4_mtp():
|
||||
if speculative_config.method == "dflash":
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
DFlashSpeculator,
|
||||
)
|
||||
|
||||
return DFlashSpeculator(vllm_config, device)
|
||||
elif speculative_config.use_gemma4_mtp():
|
||||
from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import (
|
||||
Gemma4Speculator,
|
||||
)
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
DecodeSpeculatorCudaGraphManager,
|
||||
PrefillSpeculatorCudaGraphManager,
|
||||
@@ -279,34 +278,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
def sample_draft(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
if draft_logits is not None:
|
||||
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
|
||||
# used for draft and target sampling.
|
||||
return gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
positions + 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=draft_logits,
|
||||
output_processed_logits_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
build_slot_mappings_by_layer,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionState,
|
||||
BatchExecutionDescriptor,
|
||||
CudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
def _prepare_dflash_inputs_to_capture(
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
input_buffers: InputBuffers,
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
skip_attn: bool,
|
||||
causal: bool,
|
||||
) -> AttentionState:
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, kv_cache_config
|
||||
)
|
||||
|
||||
attn_metadata = None
|
||||
if not skip_attn:
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
query_start_loc_gpu=input_batch.query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=num_tokens // num_reqs,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
max_seq_len=max_model_len,
|
||||
block_tables=input_block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
for_cudagraph_capture=True,
|
||||
causal=causal,
|
||||
)
|
||||
return AttentionState(attn_metadata, slot_mappings_by_layer)
|
||||
|
||||
|
||||
class DFlashCudaGraphManager(CudaGraphManager):
|
||||
"""DFlash CudaGraphManager for the parallel-drafting query forward,
|
||||
building its own attention metadata from scratch."""
|
||||
|
||||
def __init__(self, *args, causal: bool = False, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.causal = causal
|
||||
|
||||
def capture(
|
||||
self,
|
||||
forward_fn: Callable,
|
||||
input_buffers: InputBuffers,
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu")
|
||||
if self.dp_size > 1
|
||||
else None
|
||||
)
|
||||
attn_state = _prepare_dflash_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
input_buffers,
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
causal=self.causal,
|
||||
)
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
fwd = lambda cg_mode: forward_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cg_mode,
|
||||
)
|
||||
return fwd, attn_state
|
||||
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
@@ -0,0 +1,573 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
|
||||
get_dflash_causal,
|
||||
load_dflash_model,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DFlashSpeculator(DraftModelSpeculator):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
self.hidden_states = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
# Multimodal inputs not currently supported.
|
||||
self.supports_mm_inputs = False
|
||||
|
||||
# Each request emits exactly (bonus + N mask) query tokens per step.
|
||||
self.num_query_per_req = 1 + self.num_speculative_steps
|
||||
|
||||
self.parallel_drafting_token_id = get_parallel_drafting_token_id(
|
||||
self.draft_model_config.hf_config
|
||||
)
|
||||
|
||||
self.dflash_causal = get_dflash_causal(self.draft_model_config)
|
||||
|
||||
# Buffers for context K/V precomputation. Populated by prepare_dflash_inputs,
|
||||
# and processed by the model's precompute_and_store_context_kv method.
|
||||
# NOT captured by CUDA graphs.
|
||||
self.context_positions = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
self.context_slot_mapping = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Per-mask-token sampling buffers. Flattened from (num_reqs, num_spec_tokens).
|
||||
max_num_sampled_tokens = self.max_num_reqs * self.num_speculative_steps
|
||||
self.sample_indices = torch.zeros(
|
||||
max_num_sampled_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
self.sample_pos = torch.zeros(
|
||||
max_num_sampled_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
self.sample_idx_mapping = torch.zeros(
|
||||
max_num_sampled_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
# [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into
|
||||
# draft_logits[req, step, :].
|
||||
self.sample_col = torch.arange(
|
||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||
).repeat(self.max_num_reqs)
|
||||
|
||||
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
|
||||
self.draft_kv_cache_group_id: int = -1
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# PIECEWISE cudagraphs are not supported for dflash
|
||||
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
|
||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
else:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
self.query_cudagraph_manager = DFlashCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=self.num_query_per_req,
|
||||
causal=self.dflash_causal,
|
||||
)
|
||||
|
||||
def capture(self, attn_states: dict | None = None) -> None:
|
||||
logger.info("Capturing model for DFlash speculator...")
|
||||
# Reset sampling indices to zero to prevent stale values from prior
|
||||
# dummy runs from being baked into the captured graph.
|
||||
self.sample_indices.zero_()
|
||||
self.sample_pos.zero_()
|
||||
self.sample_idx_mapping.zero_()
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.capture(
|
||||
self._generate_draft,
|
||||
self.input_buffers,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
self.max_model_len,
|
||||
progress_bar_desc="Capturing dflash CUDA graphs",
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> nn.Module:
|
||||
return load_dflash_model(target_model, self.vllm_config)
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
) -> None:
|
||||
super().set_attn(model_state, kv_cache_config, block_tables)
|
||||
|
||||
# DFlash precomputes context K/V with a single block_size; mixing
|
||||
# kv-cache groups would silently corrupt the cache for the non-matching group.
|
||||
draft_groups = [gid for gid, g in enumerate(self.attn_groups) if g]
|
||||
assert len(draft_groups) == 1, (
|
||||
"DFlash currently requires all draft attention layers to share "
|
||||
"a single kv-cache group."
|
||||
)
|
||||
self.draft_kv_cache_group_id = draft_groups[0]
|
||||
self.draft_block_size = self.block_tables.block_sizes[
|
||||
self.draft_kv_cache_group_id
|
||||
]
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(
|
||||
self,
|
||||
num_tokens: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> torch.Tensor:
|
||||
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
|
||||
with set_forward_context(
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
):
|
||||
last_hidden_states = self.model(
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens],
|
||||
positions=self.input_buffers.positions[:num_tokens],
|
||||
inputs_embeds=None,
|
||||
)
|
||||
return last_hidden_states
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
last_hidden_states = self._run_model(
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
|
||||
num_sample = num_reqs * self.num_speculative_steps
|
||||
sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]]
|
||||
draft_tokens = self.sample_draft(
|
||||
sample_hidden_states,
|
||||
self.sample_pos[:num_sample],
|
||||
self.sample_idx_mapping[:num_sample],
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.sample_col[:num_sample],
|
||||
self.draft_logits,
|
||||
)
|
||||
self.draft_tokens[:num_reqs] = draft_tokens.view(
|
||||
num_reqs, self.num_speculative_steps
|
||||
)
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
assert num_query_per_req is None # Omitted for DFlash, read from self instead
|
||||
return super()._build_draft_attn_metadata(
|
||||
num_reqs,
|
||||
num_reqs_padded,
|
||||
num_tokens_padded,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
causal=causal,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
attn_metadata: dict[str, Any],
|
||||
slot_mappings: dict[str, torch.Tensor],
|
||||
# [num_tokens, hidden_size]
|
||||
last_hidden_states: torch.Tensor,
|
||||
# num_layers x [num_tokens, hidden_size]
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_target_tokens = input_batch.num_tokens
|
||||
num_query_tokens = num_reqs * self.num_query_per_req
|
||||
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
||||
self.draft_max_seq_len = min(
|
||||
max_seq_len + self.num_query_per_req, self.max_model_len
|
||||
)
|
||||
|
||||
# NOTE: To avoid CPU-GPU synchronization without CPU knowing the
|
||||
# number of rejected tokens, we maintain the size of input_ids and
|
||||
# hidden_states the same as the target model's. This means, we pad each
|
||||
# request's query length to include any rejected positions.
|
||||
if aux_hidden_states:
|
||||
hidden_states = self.model.combine_hidden_states(
|
||||
torch.cat(aux_hidden_states, dim=-1)
|
||||
)
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens])
|
||||
|
||||
self._copy_request_inputs(
|
||||
num_reqs,
|
||||
input_batch.idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
)
|
||||
|
||||
if dummy_run and skip_attn_for_dummy_run:
|
||||
# Memory profiling path: block_tables / kv_cache_config are not initialized.
|
||||
# Since DFlash needs to build its own attention metadata, we must skip the
|
||||
# preparation in this path and run a minimal forward pass.
|
||||
self.model.precompute_and_store_context_kv(
|
||||
self.hidden_states[:num_target_tokens],
|
||||
self.context_positions[:num_target_tokens],
|
||||
)
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
# The query slot mapping is written into the shared BlockTables slot_mappings.
|
||||
# That buffer's address is what the captured CUDA graph reads from at replay.
|
||||
assert self.draft_kv_cache_group_id >= 0
|
||||
query_slot_mapping = self.block_tables.slot_mappings[
|
||||
self.draft_kv_cache_group_id
|
||||
]
|
||||
prepare_dflash_inputs(
|
||||
self.input_buffers,
|
||||
query_slot_mapping,
|
||||
self.context_positions,
|
||||
self.context_slot_mapping,
|
||||
self.sample_indices,
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
input_batch,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
self.block_tables.input_block_tables[self.draft_kv_cache_group_id],
|
||||
self.draft_block_size,
|
||||
self.parallel_drafting_token_id,
|
||||
self.num_query_per_req,
|
||||
self.num_speculative_steps,
|
||||
self.max_num_reqs,
|
||||
self.max_num_tokens,
|
||||
)
|
||||
|
||||
# Pre-insert context K/V into the cache. Runs eagerly outside the captured graph
|
||||
# because the context shape varies per step. During dummy runs the block tables
|
||||
# are placeholders, so we skip the cache write to avoid clobbering real entries.
|
||||
self.model.precompute_and_store_context_kv(
|
||||
self.hidden_states[:num_target_tokens],
|
||||
self.context_positions[:num_target_tokens],
|
||||
context_slot_mapping=(
|
||||
None if dummy_run else self.context_slot_mapping[:num_target_tokens]
|
||||
),
|
||||
)
|
||||
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.query_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
uniform_token_count=self.num_query_per_req,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
|
||||
# Rebuild the draft attention metadata even when replaying the FULL
|
||||
# graph so that any attention metadata builder state is updated.
|
||||
draft_attn_metadata = self._build_draft_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
causal=self.dflash_causal,
|
||||
)
|
||||
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
self.block_tables.slot_mappings[:, :num_tokens_padded],
|
||||
self.kv_cache_config,
|
||||
)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
self._generate_draft(
|
||||
num_reqs_padded,
|
||||
num_tokens_padded,
|
||||
draft_attn_metadata,
|
||||
draft_slot_mappings_by_layer,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
)
|
||||
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_dflash_inputs_kernel(
|
||||
# Outputs
|
||||
out_input_ids_ptr,
|
||||
out_query_positions_ptr,
|
||||
out_query_start_loc_ptr,
|
||||
out_seq_lens_ptr,
|
||||
out_query_slot_mapping_ptr,
|
||||
out_context_positions_ptr,
|
||||
out_context_slot_mapping_ptr,
|
||||
out_sample_indices_ptr,
|
||||
out_sample_pos_ptr,
|
||||
out_sample_idx_mapping_ptr,
|
||||
# Inputs from target batch
|
||||
target_positions_ptr,
|
||||
target_query_start_loc_ptr,
|
||||
idx_mapping_ptr,
|
||||
last_sampled_ptr,
|
||||
next_prefill_tokens_ptr,
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
# Block table for slot mapping lookup.
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
# Scalars
|
||||
parallel_drafting_token_id,
|
||||
block_size,
|
||||
num_query_per_req,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
PAD_SLOT_ID: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
num_reqs = tl.num_programs(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
|
||||
|
||||
ctx_start = tl.load(target_query_start_loc_ptr + req_idx)
|
||||
ctx_end = tl.load(target_query_start_loc_ptr + req_idx + 1)
|
||||
num_ctx = ctx_end - ctx_start
|
||||
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
valid_ctx_end = ctx_end - num_rejected
|
||||
|
||||
num_sampled = tl.load(num_sampled_ptr + req_idx)
|
||||
if num_sampled > 0:
|
||||
bonus_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32)
|
||||
else:
|
||||
# Chunked prefilling: splice in the next prefill token.
|
||||
bonus_token = tl.load(next_prefill_tokens_ptr + req_state_idx).to(tl.int32)
|
||||
|
||||
last_valid_pos = tl.load(target_positions_ptr + valid_ctx_end - 1)
|
||||
query_base = req_idx * num_query_per_req
|
||||
|
||||
j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
is_ctx = j < num_ctx
|
||||
is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req)
|
||||
query_off = j - num_ctx
|
||||
|
||||
# --- Context positions / slots ---
|
||||
ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0)
|
||||
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0)
|
||||
ctx_block_num = ctx_pos // block_size
|
||||
ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1)
|
||||
ctx_block_id = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + ctx_block_num,
|
||||
mask=is_ctx,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size)
|
||||
tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx)
|
||||
tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx)
|
||||
|
||||
# --- Query positions / input_ids / slots ---
|
||||
query_pos = last_valid_pos + 1 + query_off
|
||||
query_idx = query_base + query_off
|
||||
is_bonus = is_query & (query_off == 0)
|
||||
input_id = tl.where(is_bonus, bonus_token, parallel_drafting_token_id)
|
||||
|
||||
q_block_num = query_pos // block_size
|
||||
q_block_num = tl.minimum(q_block_num, block_table_stride - 1)
|
||||
q_block_id = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + q_block_num,
|
||||
mask=is_query,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
q_slot = q_block_id * block_size + (query_pos % block_size)
|
||||
|
||||
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
|
||||
tl.store(out_query_positions_ptr + query_idx, query_pos, mask=is_query)
|
||||
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
|
||||
|
||||
# --- Sample indices / positions / idx_mapping (mask tokens only) ---
|
||||
is_sample = is_query & (query_off > 0)
|
||||
sample_idx = req_idx * num_speculative_steps + (query_off - 1)
|
||||
tl.store(out_sample_indices_ptr + sample_idx, query_idx, mask=is_sample)
|
||||
tl.store(out_sample_pos_ptr + sample_idx, query_pos, mask=is_sample)
|
||||
tl.store(out_sample_idx_mapping_ptr + sample_idx, req_state_idx, mask=is_sample)
|
||||
|
||||
if block_idx == 0:
|
||||
tl.store(out_query_start_loc_ptr + req_idx, query_base)
|
||||
# seq_lens is the absolute sequence length the draft attention
|
||||
# reads up to (context + query), not just the count of accepted
|
||||
# tokens this step.
|
||||
tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req)
|
||||
if req_idx == num_reqs - 1:
|
||||
# Pad per-request buffers to max_num_reqs for CUDA graph safety.
|
||||
last_query_end = num_reqs * num_query_per_req
|
||||
for i in range(num_reqs, max_num_reqs + 1, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs + 1
|
||||
tl.store(out_query_start_loc_ptr + block, last_query_end, mask=mask)
|
||||
for i in range(num_reqs, max_num_reqs, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_reqs
|
||||
tl.store(out_seq_lens_ptr + block, 0, mask=mask)
|
||||
# Padded sample slots point at query index 0 (a valid row in
|
||||
# last_hidden_states) so CG replay never reads OOB.
|
||||
pad_start = num_reqs * num_speculative_steps
|
||||
pad_end = max_num_reqs * num_speculative_steps
|
||||
for i in range(pad_start, pad_end, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < pad_end
|
||||
tl.store(out_sample_indices_ptr + block, 0, mask=mask)
|
||||
tl.store(out_sample_pos_ptr + block, 0, mask=mask)
|
||||
tl.store(out_sample_idx_mapping_ptr + block, 0, mask=mask)
|
||||
# Pad query slot mappings past num_query_tokens with PAD so the
|
||||
# captured CG sees PAD slots (no K/V write) for replay sizes
|
||||
# larger than the current request count.
|
||||
q_pad_start = num_reqs * num_query_per_req
|
||||
for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_tokens
|
||||
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
|
||||
|
||||
|
||||
def prepare_dflash_inputs(
|
||||
input_buffers: InputBuffers,
|
||||
query_slot_mapping: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor,
|
||||
sample_indices: torch.Tensor,
|
||||
sample_pos: torch.Tensor,
|
||||
sample_idx_mapping: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
# [num_reqs]
|
||||
num_sampled: torch.Tensor,
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs, max_num_blocks]
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
parallel_drafting_token_id: int,
|
||||
num_query_per_req: int,
|
||||
num_speculative_steps: int,
|
||||
max_num_reqs: int,
|
||||
max_num_tokens: int,
|
||||
) -> None:
|
||||
num_reqs = input_batch.num_reqs
|
||||
assert num_reqs > 0
|
||||
# Cover the longest possible per-request span (ctx + query). Use the max
|
||||
# per-request query length, not the total token count across the batch.
|
||||
max_target_query_len = int(input_batch.num_scheduled_tokens.max())
|
||||
max_tokens_per_req = max_target_query_len + num_query_per_req
|
||||
BLOCK_SIZE = min(256, triton.next_power_of_2(max(1, max_tokens_per_req)))
|
||||
num_blocks = triton.cdiv(max_tokens_per_req, BLOCK_SIZE)
|
||||
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
input_batch.positions,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.idx_mapping,
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
parallel_drafting_token_id,
|
||||
block_size,
|
||||
num_query_per_req,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
PAD_SLOT_ID=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
|
||||
|
||||
|
||||
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
|
||||
"""Whether the DFlash draft uses causal (vs non-causal) attention."""
|
||||
dflash_config = getattr(draft_model_config.hf_config, "dflash_config", None) or {}
|
||||
return dflash_config.get("causal", False)
|
||||
|
||||
|
||||
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
# Modify the attention config so that we select an attention backend that matches
|
||||
# the causal/non-causal mode of the dflash model.
|
||||
causal = get_dflash_causal(draft_model_config)
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config, use_non_causal=not causal
|
||||
),
|
||||
)
|
||||
with set_model_tag("dflash_head"):
|
||||
dflash_model = get_model(
|
||||
vllm_config=draft_vllm_config, model_config=draft_model_config
|
||||
)
|
||||
|
||||
target_language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
target_inner = target_language_model.model
|
||||
draft_inner = dflash_model.model
|
||||
|
||||
# Skip embedding sharing under PP — each rank owns its own embedding.
|
||||
if get_pp_group().world_size == 1:
|
||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||
target_inner, "embedding", None
|
||||
)
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
dflash_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
# Share lm_head with the target unless the draft remaps vocab via
|
||||
# draft_id_to_target_id (in which case its own lm_head is required).
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
if (
|
||||
target_lm_head is not None
|
||||
and draft_lm_head is not None
|
||||
and getattr(dflash_model, "draft_id_to_target_id", None) is None
|
||||
):
|
||||
del dflash_model.lm_head
|
||||
dflash_model.lm_head = target_lm_head
|
||||
|
||||
return dflash_model
|
||||
@@ -38,9 +38,12 @@ def get_eagle3_aux_layers_from_config(
|
||||
if not (spec_config and spec_config.draft_model_config):
|
||||
return None
|
||||
hf_config = spec_config.draft_model_config.hf_config
|
||||
if not hasattr(hf_config, "eagle_aux_hidden_state_layer_ids"):
|
||||
return None
|
||||
layer_ids = hf_config.eagle_aux_hidden_state_layer_ids
|
||||
layer_ids = getattr(hf_config, "eagle_aux_hidden_state_layer_ids", None)
|
||||
if not layer_ids:
|
||||
dflash_config = getattr(hf_config, "dflash_config", None)
|
||||
if dflash_config and isinstance(dflash_config, dict):
|
||||
# Add 1 to convert DFlash's aux layer id semantics
|
||||
layer_ids = [i + 1 for i in (dflash_config.get("target_layer_ids") or [])]
|
||||
if layer_ids and isinstance(layer_ids, (list, tuple)):
|
||||
return tuple(layer_ids)
|
||||
return None
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
|
||||
|
||||
class BaseSpeculator(ABC):
|
||||
@@ -178,9 +179,15 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
num_query_per_req: int = 1,
|
||||
causal: bool = True,
|
||||
) -> dict[str, Any] | None:
|
||||
query_start_loc_cpu = torch.clamp(
|
||||
self.arange[: num_reqs_padded + 1], max=num_reqs
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
# attention backends require.
|
||||
query_start_loc_cpu = (
|
||||
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
|
||||
* num_query_per_req
|
||||
)
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
@@ -194,15 +201,44 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
: num_reqs_padded + 1
|
||||
],
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=1,
|
||||
max_query_len=num_query_per_req,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
max_seq_len=self.draft_max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
causal=causal,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def sample_draft(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
if draft_logits is not None:
|
||||
# NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise
|
||||
# used for draft and target sampling.
|
||||
return gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
positions + 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=draft_logits,
|
||||
output_processed_logits_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
def _copy_request_inputs(
|
||||
self,
|
||||
num_reqs: int,
|
||||
|
||||
@@ -45,3 +45,22 @@ class DraftTokensHandler:
|
||||
# This case only happens when async scheduling is disabled.
|
||||
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
|
||||
return DraftTokenIds(self.req_ids, draft_token_ids)
|
||||
|
||||
|
||||
def get_parallel_drafting_token_id(hf_config) -> int:
|
||||
"""Resolve the mask token id used for parallel drafting slots.
|
||||
|
||||
Checks (in order): `dflash_config.mask_token_id`, `pard_token`,
|
||||
`ptd_token_id`. Raises ValueError if none are present.
|
||||
"""
|
||||
dflash_config = getattr(hf_config, "dflash_config", None) or {}
|
||||
if "mask_token_id" in dflash_config:
|
||||
return int(dflash_config["mask_token_id"])
|
||||
if hasattr(hf_config, "pard_token"):
|
||||
return int(hf_config.pard_token)
|
||||
if hasattr(hf_config, "ptd_token_id"):
|
||||
return int(hf_config.ptd_token_id)
|
||||
raise ValueError(
|
||||
"Model config must specify `dflash_config.mask_token_id`,"
|
||||
" `pard_token`, or `ptd_token_id` for parallel drafting."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user