[MRV2][SD] Make Dynamic SD comatible with Full Cuda Graphs (#45953)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Ekagra Ranjan
2026-07-03 23:58:27 -04:00
committed by GitHub
co-authored by Benjamin Chislett Nick Hill
parent 67ff0ae30f
commit 07516fda67
8 changed files with 436 additions and 36 deletions
@@ -71,8 +71,5 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \
## Limitations
* only tested with Eagle and Eagle-3. Other SD methods may or may not work out of the box
* only usable with Model Runner V1
* not compatible with full cuda graph so we force piece-wise cuda graph with this feature
We are working on enabling it on MRv2 with full cuda graph support.
* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box
* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature
+31
View File
@@ -34,6 +34,7 @@ from vllm.config.vllm import (
OptimizationLevel,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionCGSupport
DEVICE_TYPE = current_platform.device_type
@@ -67,6 +68,36 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
@pytest.mark.parametrize(
("use_v2_model_runner", "expected_capture_sizes"),
[
(False, [4, 8, 12, 16]),
(True, list(range(1, 17))),
],
)
def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
use_v2_model_runner,
expected_capture_sizes,
):
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
cudagraph_capture_sizes=list(range(1, 17)),
)
compilation_config.max_cudagraph_capture_size = 16
compilation_config.post_init_cudagraph_sizes()
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=4,
use_v2_model_runner=use_v2_model_runner,
tensor_parallel_size=1,
)
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
@pytest.mark.parametrize(
("model_config", "expected"),
[
+328
View File
@@ -0,0 +1,328 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config import (
CompilationConfig,
CUDAGraphMode,
ParallelConfig,
SchedulerConfig,
VllmConfig,
)
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
pytestmark = pytest.mark.cpu_test
def _create_vllm_config_for_dsd(
max_num_seqs: int,
max_spec_tokens: int,
*,
cudagraph_mode: str = "FULL_AND_PIECEWISE",
use_dynamic_sd: bool = True,
num_spec_per_batch_size: list[tuple[int, int, int]] | None = None,
) -> MagicMock:
"""Create a minimal config that exercises DSD cudagraph dispatch.
The test uses an exact capture-size grid so that every valid uniform decode
shape has a directly matching FULL graph candidate.
``num_spec_per_batch_size`` lets a test supply an explicit DSD schedule of
``(range_start, range_end, num_speculative_tokens)`` tuples. When omitted,
a schedule covering every query length in ``[1, max_decode_query_len]`` is
generated.
"""
max_decode_query_len = max_spec_tokens + 1
max_capture_tokens = max_num_seqs * max_decode_query_len
compilation_config = CompilationConfig(
cudagraph_mode=cudagraph_mode,
cudagraph_capture_sizes=list(range(1, max_capture_tokens + 1)),
)
compilation_config.max_cudagraph_capture_size = max_capture_tokens
compilation_config.post_init_cudagraph_sizes()
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.compilation_config = compilation_config
vllm_config.scheduler_config = SchedulerConfig.default_factory(
max_num_seqs=max_num_seqs,
)
vllm_config.parallel_config = ParallelConfig()
# num_speculative_tokens is the max K (num_speculative_steps). The manager
# recovers num_new_sampled_tokens_per_step as
# decode_query_len - num_speculative_tokens; with decode_query_len =
# max_spec_tokens + 1 this yields the normal per-step bonus of 1.
vllm_config.num_speculative_tokens = max_spec_tokens
speculative_config = MagicMock()
speculative_config.uses_dynamic_speculative_decoding.return_value = use_dynamic_sd
if use_dynamic_sd:
# DSD reads the per-batch-size schedule; a schedule entry with K
# speculative tokens maps to decode query length K + 1. By default
# provide every query length in [1, max_decode_query_len] (i.e. K in
# [0, max_spec_tokens]) so the manager captures a FULL decode graph for
# each uniform shape.
if num_spec_per_batch_size is None:
num_spec_per_batch_size = [
(qlen, qlen, qlen - 1) for qlen in range(1, max_decode_query_len + 1)
]
speculative_config.num_speculative_tokens_per_batch_size = (
num_spec_per_batch_size
)
else:
speculative_config.num_speculative_tokens_per_batch_size = None
vllm_config.speculative_config = speculative_config
return vllm_config
def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch):
"""Dynamic SD should create FULL decode candidates for every k in [1, K+1].
This validates the MRv2 CudaGraphManager path directly: once candidate
shapes have been built, dispatch() should pick a FULL graph for every
uniform decode batch shape produced by DSD up to max_num_seqs.
"""
max_num_seqs = 512
max_spec_tokens = 7
max_decode_query_len = max_spec_tokens + 1
# CudaGraphManager consults PP rank helpers during initialization even
# though this test only exercises CPU-side candidate generation.
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=max_decode_query_len,
)
# dispatch() only uses the precomputed candidate table after graphs are
# considered captured. The actual graph objects are irrelevant here.
manager._graphs_captured = True
for num_reqs in range(1, max_num_seqs + 1):
for max_query_len in range(1, max_decode_query_len + 1):
# Uniform decode means every request contributes the same number of
# tokens, so the total token count is exactly num_reqs * query_len.
num_tokens = num_reqs * max_query_len
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
num_reqs,
num_tokens,
max_query_len,
)
# The scheduler should mark every one of these shapes as a uniform
# decode batch, which is what enables FULL decode graph selection.
assert uniform_tok_count == max_query_len
desc = manager.dispatch(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
# With DSD enabled, MRv2 should have captured a FULL candidate for
# every k in [1, K+1], so dispatch should stay on the FULL path.
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.uniform_token_count == max_query_len
assert desc.num_tokens == num_tokens
assert desc.num_reqs == num_reqs
assert desc.num_active_loras == 0
def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
"""DSD should use PIECEWISE when the batch is not a uniform decode batch.
FULL DSD graphs are captured separately for each decode query length k.
When runtime tokens are not uniform, uniform_token_count is None and those
FULL candidates should be skipped in favor of the mixed-batch PIECEWISE
graph under FULL_AND_PIECEWISE mode.
"""
max_spec_tokens = 4
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=512,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
use_dynamic_sd=True,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=max_spec_tokens + 1,
)
manager._graphs_captured = True
# This shape is intentionally non-uniform: 3 tokens across 2 requests
# cannot correspond to a single per-request query length.
desc = manager.dispatch(
num_reqs=2,
num_tokens=3,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_reqs is None
assert desc.num_tokens == 3
assert desc.num_active_loras == 0
def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
"""Without DSD, only the max decode query length should get FULL graphs.
Basic SD captures FULL decode graphs only for decode_query_len = K + 1.
Uniform batches with smaller query lengths should therefore miss the FULL
path entirely when using FULL_AND_PIECEWISE.
"""
max_num_seqs = 512
max_spec_tokens = 7
max_decode_query_len = max_spec_tokens + 1
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
use_dynamic_sd=False,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=max_decode_query_len,
)
manager._graphs_captured = True
for num_reqs in range(1, max_num_seqs + 1):
for max_query_len in range(1, max_decode_query_len):
# These are still uniform decode batches, but basic SD should only
# have FULL graphs for query_len == max_decode_query_len.
num_tokens = num_reqs * max_query_len
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
num_reqs,
num_tokens,
max_query_len,
)
assert uniform_tok_count == max_query_len
desc = manager.dispatch(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_tokens == num_tokens
assert desc.num_reqs is None
assert desc.num_active_loras == 0
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
"""DSD should only capture FULL graphs for query lengths in the schedule.
With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the
scheduled speculative-token counts (K = 4 and K = 3) become decode query
lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get
FULL graphs, while every other query length (e.g. the lower values 1, 2, 3)
must fall back to the mixed-batch PIECEWISE graph.
"""
max_num_seqs = 128
max_spec_tokens = 7
max_decode_query_len = max_spec_tokens + 1
# (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are
# scheduled, so FULL decode graphs should exist for query lengths K + 1,
# i.e. exactly {5, 4}.
num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)]
scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size}
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
use_dynamic_sd=True,
num_spec_per_batch_size=num_spec_per_batch_size,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=max_decode_query_len,
)
manager._graphs_captured = True
for num_reqs in range(1, max_num_seqs + 1):
for max_query_len in range(1, max_decode_query_len + 1):
num_tokens = num_reqs * max_query_len
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
num_reqs,
num_tokens,
max_query_len,
)
assert uniform_tok_count == max_query_len
desc = manager.dispatch(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
if max_query_len in scheduled_query_lens:
# Scheduled query lengths get a dedicated FULL decode graph.
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.uniform_token_count == max_query_len
assert desc.num_tokens == num_tokens
assert desc.num_reqs == num_reqs
else:
# Unscheduled query lengths (including the lower values 1 and 2)
# have no FULL candidate and must fall back to PIECEWISE.
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_tokens == num_tokens
assert desc.num_reqs is None
assert desc.num_active_loras == 0
+6 -2
View File
@@ -1321,6 +1321,7 @@ class CompilationConfig:
min_cg_support: "AttentionCGSupport",
min_cg_attn_backend: str | None,
uniform_decode_query_len: int = 1,
use_v2_model_runner: bool = False,
tensor_parallel_size: int = 1,
kv_cache_config: "KVCacheConfig | None" = None,
max_num_reqs: int | None = None,
@@ -1421,13 +1422,16 @@ class CompilationConfig:
"and make sure compilation mode is VLLM_COMPILE"
)
# Adjust cudagraph sizes to be a multiple of uniform_decode_query_len
# MRV1 adjusts cudagraph sizes to be a multiple of uniform_decode_query_len
# to avoid: https://github.com/vllm-project/vllm/issues/28207 and temp-fix:
# https://github.com/vllm-project/vllm/issues/28207#issuecomment-3504004536
# Will be removed in the near future when we have separate cudagraph capture
# sizes for decode and mixed prefill-decode.
# MRV2 handles cudagraph capture sizing in cudagraph_utils.py
# and doesn't need below: https://github.com/vllm-project/vllm/pull/45953
if (
cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
not use_v2_model_runner
and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
and uniform_decode_query_len > 1
):
self.adjust_cudagraph_sizes_for_spec_decode(
+3 -4
View File
@@ -774,13 +774,15 @@ class VllmConfig:
speculative_config is None
or not speculative_config.uses_dynamic_speculative_decoding()
or not self.compilation_config.cudagraph_mode.has_full_cudagraphs()
or self.use_v2_model_runner
):
return
logger.warning_once(
"Dynamic speculative decoding changes the target verification "
"length at runtime. Overriding cudagraph_mode from %s to "
"PIECEWISE for reliability.",
"PIECEWISE for reliability. Use VLLM_USE_V2_MODEL_RUNNER=1 "
"if you want to use full CUDA graphs.",
self.compilation_config.cudagraph_mode.name,
)
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
@@ -2062,9 +2064,6 @@ class VllmConfig:
):
unsupported.append(f"speculative method '{speculative_config.method}'")
if speculative_config.uses_dynamic_speculative_decoding():
unsupported.append("dynamic speculative decoding")
# V2 EagleSpeculator does not support parallel_drafting (for P-Eagle).
# DFlash and DSpark use parallel drafting natively in V2 via their
# own speculators.
+58 -19
View File
@@ -27,6 +27,7 @@ from vllm.logger import init_logger
from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import round_up
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
@@ -141,10 +142,7 @@ class CudaGraphManager:
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
# adjust the cudagraph sizes to be a multiple of the uniform decode query length
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
self.decode_query_len, self.tp_size
)
self._init_candidates()
# Breakable CUDA graph (PW CUDA graph without torch.compile)
@@ -191,31 +189,72 @@ class CudaGraphManager:
decode_mode = self.cudagraph_mode.decode_mode()
mixed_mode = self.cudagraph_mode.mixed_mode()
separate_decode_routine = self.cudagraph_mode.separate_routine()
max_cg_capture_size = self.compilation_config.max_cudagraph_capture_size
descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = (
defaultdict(list)
)
descs_by_mode = defaultdict(list)
descs_by_mode: defaultdict[CUDAGraphMode, list[BatchExecutionDescriptor]] = (
defaultdict(list)
)
# When using Dynamic SD, num_speculative_tokens is the max number of
# draft tokens. The scheduler might use a smaller number so we need
# to capture graphs for all possible values during decode.
speculative_config = self.vllm_config.speculative_config
if (
speculative_config
and speculative_config.uses_dynamic_speculative_decoding()
):
num_spec_per_batch_size = (
speculative_config.num_speculative_tokens_per_batch_size
)
# uses_dynamic_speculative_decoding() guarantees this is set.
assert num_spec_per_batch_size is not None
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
# _per_step. Recover num_new_sampled_tokens_per_step
# from the values the manager already has.
num_new_sampled_tokens_per_step = (
self.decode_query_len - self.vllm_config.num_speculative_tokens
)
# Each entry is (range_start, range_end, num_speculative_tokens).
decode_query_lens = [
x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size
]
else:
decode_query_lens = [self.decode_query_len]
for num_tokens, num_active_loras in product(
capture_sizes, self.lora_capture_cases
):
# Capture uniform decode specfifc graphs if required
# (i.e. separate decode routine)
if (
separate_decode_routine
and decode_mode
and self.decode_query_len <= num_tokens <= max_decode_tokens
):
desc = BatchExecutionDescriptor(
cg_mode=decode_mode,
num_tokens=num_tokens,
num_reqs=num_tokens // self.decode_query_len,
uniform_token_count=self.decode_query_len,
num_active_loras=num_active_loras,
)
descs_by_mode[decode_mode].append(desc)
descs_by_token_lora[(num_tokens, num_active_loras)].append(desc)
if separate_decode_routine and decode_mode:
for decode_query_len in decode_query_lens:
rounded_num_tokens = round_up(num_tokens, decode_query_len)
rounded_num_reqs = rounded_num_tokens // decode_query_len
if (
rounded_num_tokens > max_decode_tokens
or rounded_num_tokens > max_cg_capture_size
or rounded_num_reqs > self.max_num_reqs
):
continue
desc = BatchExecutionDescriptor(
cg_mode=decode_mode,
num_tokens=rounded_num_tokens,
num_reqs=rounded_num_reqs,
uniform_token_count=decode_query_len,
num_active_loras=num_active_loras,
)
# avoid duplicate graphs
if desc not in descs_by_mode[decode_mode]:
descs_by_mode[decode_mode].append(desc)
descs_by_token_lora[
(rounded_num_tokens, num_active_loras)
].append(desc)
if mixed_mode:
# for PIECEWISE graphs there is no limit on requests when replaying
+4 -3
View File
@@ -454,9 +454,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
attn_cg_support.min_cg_support,
attn_cg_support.min_cg_attn_backend,
self.decode_query_len,
self.parallel_config.tensor_parallel_size,
self.kv_cache_config,
self.max_num_reqs,
use_v2_model_runner=True,
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
kv_cache_config=self.kv_cache_config,
max_num_reqs=self.max_num_reqs,
)
self.cudagraph_manager = ModelCudaGraphManager(
self.vllm_config,
+4 -3
View File
@@ -6930,9 +6930,10 @@ class GPUModelRunner(
min_cg_support,
min_cg_attn_backend,
self.uniform_decode_query_len,
self.parallel_config.tensor_parallel_size,
self.kv_cache_config,
self.max_num_reqs,
use_v2_model_runner=False,
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
kv_cache_config=self.kv_cache_config,
max_num_reqs=self.max_num_reqs,
is_profiling=is_profiling,
)
# Trigger cudagraph dispatching keys initialization after