[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-05 14:54:11 -04:00
committed by GitHub
parent c32495d58f
commit 74c5cbcd7b
15 changed files with 190 additions and 18 deletions
@@ -1,11 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass, replace
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
@@ -17,6 +19,11 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup
@dataclass(frozen=True)
class SpeculatorBatchDescriptor(BatchExecutionDescriptor):
num_speculative_tokens: int = 0
class SpeculatorCudaGraphManager(CudaGraphManager):
"""CudaGraphManager for draft prefill and decode.
@@ -27,6 +34,18 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
earlier capture would execute kernels with stale buffer contents.
"""
def specialize_spec_tokens(
self, desc: BatchExecutionDescriptor, num_speculative_tokens: int
) -> BatchExecutionDescriptor:
if desc.cg_mode != CUDAGraphMode.FULL or desc in self.graphs:
return desc
desc = SpeculatorBatchDescriptor(
**vars(desc), num_speculative_tokens=num_speculative_tokens
)
return (
desc if desc in self.graphs else replace(desc, cg_mode=CUDAGraphMode.NONE)
)
def capture(
self,
forward_fn: Callable,
@@ -36,7 +55,25 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
progress_bar_desc: str = "Capturing CUDA graphs",
specialize_spec_tokens: bool = False,
) -> None:
if specialize_spec_tokens:
speculative_config = self.vllm_config.speculative_config
assert speculative_config is not None
dense_schedule = build_dynamic_sd_schedule_lookup(
speculative_config.num_speculative_tokens_per_batch_size,
vllm_max_batch_size=self.max_num_reqs,
vllm_num_speculative_tokens=self.vllm_config.num_speculative_tokens,
)
self._capture_descs = {
mode: [
SpeculatorBatchDescriptor(**vars(desc), num_speculative_tokens=k)
for desc in descs
for k in sorted(set(dense_schedule) - {0, 1}, reverse=True)
]
for mode, descs in self._capture_descs.items()
}
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
@@ -60,6 +97,13 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
max_query_len=desc.max_query_len or desc.uniform_token_count,
)
forward_kwargs = {}
if self.decode_query_len == 1:
forward_kwargs["num_speculative_steps"] = (
desc.num_speculative_tokens
if isinstance(desc, SpeculatorBatchDescriptor)
else self.vllm_config.num_speculative_tokens
)
return lambda cg_mode: forward_fn(
num_reqs,
num_tokens,
@@ -67,6 +111,7 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
slot_mappings,
num_tokens_across_dp,
cg_mode,
**forward_kwargs,
)
super().capture(create_forward_fn, progress_bar_desc)
@@ -190,6 +190,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.attn_groups,
self.kv_cache_config,
progress_bar_desc="Capturing decode CUDA graphs",
specialize_spec_tokens=self.use_fused_multi_step_decode
and self.speculative_config.uses_dynamic_speculative_decoding(),
)
self.on_multi_step_decode_end(self.max_num_reqs)
@@ -220,14 +222,17 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
num_speculative_tokens: int | None = None,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
if num_speculative_tokens is None:
num_speculative_tokens = self.num_speculative_steps
num_tokens = input_batch.num_tokens
num_tokens_padded = input_batch.num_tokens_after_padding
num_reqs = input_batch.num_reqs
max_query_len = input_batch.num_scheduled_tokens.max()
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
self.draft_max_seq_len = min(
max_seq_len + self.num_speculative_steps, self.max_model_len
max_seq_len + num_speculative_tokens, self.max_model_len
)
# NOTE(woosuk): To avoid CPU-GPU synchronization without CPU knowing the
@@ -322,9 +327,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
)
self.on_prefill_end(num_reqs)
if self.num_speculative_steps == 1:
# Early exit.
return self.draft_tokens[:num_reqs, :1]
if num_speculative_tokens <= 1:
if num_speculative_tokens == 0:
self.draft_tokens[:num_reqs].fill_(-1)
return self.draft_tokens[:num_reqs]
if self.pcp_manager is not None and not dummy_run:
self.block_tables.gather_block_tables(
@@ -360,6 +366,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
need_eager=is_profile,
dp_sync=decode_batch_sync,
)
if self.decode_cudagraph_manager is not None:
decode_batch_desc = self.decode_cudagraph_manager.specialize_spec_tokens(
decode_batch_desc, num_speculative_tokens
)
num_tokens_across_dp = (
decode_batch_sync.num_tokens_across_dp
if decode_batch_sync is not None
@@ -379,6 +389,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
decode_batch_desc,
num_tokens_across_dp,
input_batch.seq_lens_cpu_upper_bound,
num_speculative_tokens,
)
self.on_multi_step_decode_end(num_reqs)
@@ -505,6 +516,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
batch_desc: BatchExecutionDescriptor,
num_tokens_across_dp: torch.Tensor | None,
seq_lens_cpu_upper_bound: torch.Tensor,
num_speculative_steps: int,
) -> None:
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
@@ -512,7 +524,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
attn_metadata = None
slot_mappings_by_layer = None
for step in range(1, self.num_speculative_steps):
for step in range(1, num_speculative_steps):
# Rebuild every step when positions advance, or just once
# on the first step when positions are constant (Gemma4 MTP).
if not skip_attn and (self.advance_draft_positions or step == 1):
@@ -546,6 +558,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
slot_mappings_by_layer,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=batch_desc.cg_mode,
num_speculative_steps=num_speculative_steps,
)
def _fused_multi_step_decode(
@@ -555,6 +568,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
batch_desc: BatchExecutionDescriptor,
num_tokens_across_dp: torch.Tensor | None,
seq_lens_cpu_upper_bound: torch.Tensor,
num_speculative_steps: int,
) -> None:
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
@@ -593,6 +607,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
slot_mappings_by_layer,
num_tokens_across_dp,
batch_desc.cg_mode,
num_speculative_steps,
)
def _generate_fused_drafts(
@@ -602,7 +617,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
cudagraph_runtime_mode: CUDAGraphMode,
num_speculative_steps: int,
) -> None:
idx_mapping = self.idx_mapping[:num_reqs]
positions = self.input_buffers.positions[:num_reqs]
@@ -613,7 +629,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
else []
)
for step in range(1, self.num_speculative_steps):
for step in range(1, num_speculative_steps):
self.current_draft_step.fill_(step)
self._generate_draft(
num_reqs,
@@ -622,9 +638,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
num_speculative_steps,
)
if (
step < self.num_speculative_steps - 1
step < num_speculative_steps - 1
and attn_metadata is not None
and self.advance_draft_positions
):
@@ -644,7 +661,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
cudagraph_runtime_mode: CUDAGraphMode,
num_speculative_steps: int,
) -> None:
self._prepare_eplb_forward(num_reqs)
@@ -682,7 +700,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.sample_src_positions,
num_reqs,
self.max_model_len,
self.num_speculative_steps,
num_speculative_steps,
advance_draft_positions=self.advance_draft_positions,
)
@@ -360,6 +360,7 @@ class DFlashSpeculator(DraftModelSpeculator):
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
num_speculative_tokens: int | None = None,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_target_tokens = input_batch.num_tokens
@@ -355,6 +355,7 @@ class PlainDraftModelSpeculator(DraftModelSpeculator):
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
num_speculative_tokens: int | None = None,
) -> torch.Tensor:
assert self.model is not None
@@ -103,6 +103,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
num_speculative_tokens: int | None = None,
) -> torch.Tensor:
del (
last_hidden_states,
@@ -155,6 +155,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
num_speculative_tokens: int | None = None,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
@@ -97,6 +97,7 @@ class BaseSpeculator(ABC):
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
num_speculative_tokens: int | None = None,
) -> torch.Tensor:
pass