[Bugfix][Spec Decode] Capture the widest uniform decode batch by default (#50488)

Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com>
Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Signed-off-by: Rahul Chalamala <rahul@modal.com>
Co-authored-by: Janelle Cai <janelle.cai@modal.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Devin <devin@cognition.ai>
This commit is contained in:
Rahul Chalamala
2026-08-28 22:38:13 -07:00
committed by GitHub
co-authored by Janelle Cai Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Devin
parent a758a9f671
commit d3d79ffc1e
6 changed files with 621 additions and 30 deletions
+441
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
@@ -15,13 +16,16 @@ from vllm.compilation.passes.utility.fix_functionalization import (
from vllm.config import (
CompilationConfig,
CUDAGraphMode,
ModelConfig,
ParallelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.config.compilation import CompilationMode, PassConfig
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
_is_torch_equal_or_newer,
is_torch_equal,
@@ -497,6 +501,443 @@ def test_cudagraph_sizes_post_init(
)
def _mock_config_for_cudagraph_sizes(
max_num_seqs: int,
num_speculative_tokens: int,
max_num_batched_tokens: int,
compilation_config: CompilationConfig,
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
) -> MagicMock:
"""Mock VllmConfig wired up enough to run `_set_cudagraph_sizes`.
`num_speculative_tokens` and `uniform_decode_query_len` are filled in by
calling the real property functions, so the tests below cover the
derivation from `speculative_config` and not just the arithmetic
downstream of it.
"""
config = MagicMock(spec=VllmConfig)
config.compilation_config = compilation_config
config.scheduler_config = SchedulerConfig.default_factory(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=max_num_batched_tokens,
)
config.parallel_config = ParallelConfig()
config.model_config = MagicMock()
config.model_config.enforce_eager = False
config.performance_mode = None
config.diffusion_config = None
schedule = num_speculative_tokens_per_batch_size
config.speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
num_speculative_tokens_per_batch_size=schedule,
uses_dynamic_speculative_decoding=lambda: schedule is not None,
)
if num_speculative_tokens
else None
)
config.num_speculative_tokens = VllmConfig.num_speculative_tokens.fget(config)
config.uniform_decode_query_len = VllmConfig.uniform_decode_query_len.fget(config)
return config
@pytest.mark.parametrize(
("max_num_seqs", "num_speculative_tokens"),
[
# No speculation: the 2x headroom under the platform ceiling, unchanged.
(8, 0),
(32, 0),
# Speculating, but the widest decode batch still fits under the ceiling.
(64, 7),
(256, 1),
# Widest decode batch above the ceiling, which must not cut below it.
(32, 16),
(64, 16),
# Widest decode batch off the capture stride, above the ceiling.
(33, 16),
# ... and off the stride while below it, where the ceiling stands but
# the generated sizes would otherwise stop at 400.
(24, 16),
],
)
def test_default_cudagraph_capture_size_covers_widest_uniform_decode(
max_num_seqs, num_speculative_tokens
):
"""The widest uniform decode batch has to be inside the captured range.
A decode step presents up to `max_num_seqs * (1 + num_speculative_tokens)`
tokens. If that size is not captured the decode path silently falls back to
eager while the resolved mode still reports full decode graphs, so neither
the platform's token-denominated ceiling nor the capture stride may stop
short of it.
"""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
)
VllmConfig._set_cudagraph_sizes(config)
decode_query_len = 1 + num_speculative_tokens
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
token_grid_max = min(
max_num_seqs * decode_query_len * 2,
default_max_graph_size,
)
widest_uniform_decode = max_num_seqs * decode_query_len
expected_max_size = max(token_grid_max, widest_uniform_decode)
assert compilation_config.max_cudagraph_capture_size == expected_max_size
assert (
compilation_config.max_cudagraph_capture_size
== compilation_config.cudagraph_capture_sizes[-1]
)
assert widest_uniform_decode in compilation_config.cudagraph_capture_sizes
def test_default_cudagraph_capture_sizes_keep_token_grid_bounded():
"""Keep the token grid bounded; only request counts may exceed the default.
This guards the measured 581 versus 100 capture-size regression at
max_num_seqs=512 with 16 draft tokens.
"""
max_num_seqs = 512
decode_query_len = 17
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=max_num_seqs,
num_speculative_tokens=16,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=False,
):
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
VllmConfig._set_cudagraph_sizes(config)
token_grid_max = min(max_num_seqs * decode_query_len * 2, default_max_graph_size)
token_grid = [size for size in [1, 2, 4] if size <= token_grid_max]
token_grid += list(range(8, min(token_grid_max + 1, 256), 8))
token_grid += list(range(256, token_grid_max + 1, 16))
max_request_count = min(max_num_seqs, default_max_graph_size)
request_counts = [count for count in [1, 2, 4] if count <= max_request_count]
request_counts += list(range(8, min(max_request_count + 1, 256), 8))
request_counts += list(range(256, max_request_count + 1, 16))
request_counts.append(max_request_count)
expected_sizes = sorted(
set(token_grid + [count * decode_query_len for count in request_counts])
)
assert compilation_config.cudagraph_capture_sizes == expected_sizes
assert all(
size <= default_max_graph_size or size % decode_query_len == 0
for size in compilation_config.cudagraph_capture_sizes
)
def test_cudagraph_capture_sizes_respect_sequence_parallelism():
"""Sequence-parallel capture sizes stay divisible by tensor parallel size."""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
compilation_config.pass_config.enable_sp = True
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=32,
num_speculative_tokens=16,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
)
config.parallel_config = SimpleNamespace(tensor_parallel_size=2)
config.update_sizes_for_sequence_parallelism = lambda sizes: (
VllmConfig.update_sizes_for_sequence_parallelism(config, sizes)
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=False,
):
VllmConfig._set_cudagraph_sizes(config)
assert all(size % 2 == 0 for size in compilation_config.cudagraph_capture_sizes)
assert 544 in compilation_config.cudagraph_capture_sizes
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
def test_real_vllm_config_captures_widest_ngram_decode_batch():
"""Real config post-init preserves the widest static ngram decode batch."""
model_config = ModelConfig(model="facebook/opt-125m")
speculative_config = SpeculativeConfig(
prompt_lookup_min=1,
prompt_lookup_max=1,
num_speculative_tokens=16,
method="ngram",
)
scheduler_config = SchedulerConfig.default_factory(
max_num_seqs=32,
max_num_batched_tokens=32768,
)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=False,
):
config = VllmConfig(
model_config=model_config,
speculative_config=speculative_config,
scheduler_config=scheduler_config,
compilation_config=compilation_config,
)
assert 544 in config.compilation_config.cudagraph_capture_sizes
assert (
config.compilation_config.max_cudagraph_capture_size
== config.compilation_config.cudagraph_capture_sizes[-1]
== 544
)
@pytest.mark.parametrize("max_num_seqs", [8, 32, 256, 300, 512, 600, 1024, 2048])
def test_default_cudagraph_capture_size_unchanged_without_speculation(max_num_seqs):
"""Without speculation the default must reproduce the historical formula.
The platform ceiling bounds a request count, and without speculation a
request is one token, so `min(max_num_seqs, ceiling) * 1` can never lift it.
The result must match `min(max_num_seqs * 2, ceiling)` bit for bit.
"""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=max_num_seqs,
num_speculative_tokens=0,
max_num_batched_tokens=1_000_000,
compilation_config=compilation_config,
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=False,
):
VllmConfig._set_cudagraph_sizes(config)
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
assert compilation_config.max_cudagraph_capture_size == min(
max_num_seqs * 2,
default_max_graph_size,
)
def test_default_cudagraph_capture_size_covers_a_single_speculative_token():
"""One speculative token is already enough to lose the widest batch.
MTP at depth 1 gives a query length of 2, so 300 requests is a 600-token
decode batch against the platform's token ceiling. Gating the unit conversion on a
deeper speculative width would leave this common configuration dispatching
its largest decode steps eager.
"""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=300,
num_speculative_tokens=1,
max_num_batched_tokens=1_000_000,
compilation_config=compilation_config,
)
VllmConfig._set_cudagraph_sizes(config)
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
token_grid_max = min(300 * 2 * 2, default_max_graph_size)
widest_uniform_decode = 300 * 2
expected_max_size = max(token_grid_max, widest_uniform_decode)
assert compilation_config.max_cudagraph_capture_size == expected_max_size
assert (
compilation_config.max_cudagraph_capture_size
== compilation_config.cudagraph_capture_sizes[-1]
)
assert 600 in compilation_config.cudagraph_capture_sizes
def test_default_cudagraph_capture_size_caps_requests_not_tokens():
"""The ceiling admits at most the platform default number of requests.
`max_num_seqs` far above the platform default must not push the capture
range up without bound just because each request is now several tokens wide.
"""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=1024,
num_speculative_tokens=16,
max_num_batched_tokens=1_000_000,
compilation_config=compilation_config,
)
VllmConfig._set_cudagraph_sizes(config)
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
decode_query_len = 17
token_grid_max = min(1024 * decode_query_len * 2, default_max_graph_size)
widest_covered_decode = min(1024, default_max_graph_size) * decode_query_len
expected_max_size = max(token_grid_max, widest_covered_decode)
assert compilation_config.max_cudagraph_capture_size == expected_max_size
assert (
compilation_config.max_cudagraph_capture_size
== compilation_config.cudagraph_capture_sizes[-1]
)
assert expected_max_size == default_max_graph_size * 17
def _widest_covered_request_count(capture_sizes, query_len, max_num_seqs):
"""Widest request count `CudaGraphManager` can build a decode graph for.
Mirrors `_init_candidates`: a capture size is rounded up to a multiple of
the tier's query length, then dropped once the implied request count runs
past `max_num_seqs`.
"""
reachable = [
cdiv(size, query_len)
for size in capture_sizes
if cdiv(size, query_len) <= max_num_seqs
]
return max(reachable, default=0)
def test_default_cudagraph_capture_sizes_cover_every_dynamic_decode_width():
"""Each scheduled draft width needs sizes over the range it applies to.
Dynamic speculative decoding picks the width from the batch size, so
scaling by the widest one alone leaves the narrower tiers short: sizes
built from query length 17 round up to multiples of 3 that imply more
requests than the scheduler can run, and coverage stops at 227 of 256.
"""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=256,
num_speculative_tokens=16,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
# 16 draft tokens up to a batch of 16, then 2 out to 256.
num_speculative_tokens_per_batch_size=[(1, 16, 16), (17, 256, 2)],
)
VllmConfig._set_cudagraph_sizes(config)
sizes = compilation_config.cudagraph_capture_sizes
# The wide tier only ever runs to a batch of 16, the narrow one to 256.
assert _widest_covered_request_count(sizes, 17, 256) >= 16
assert _widest_covered_request_count(sizes, 3, 256) == 256
def test_dynamic_decode_capture_clamps_configured_width():
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=16,
num_speculative_tokens=3,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
num_speculative_tokens_per_batch_size=[(1, 16, 5)],
)
VllmConfig._set_cudagraph_sizes(config)
sizes = compilation_config.cudagraph_capture_sizes
assert _widest_covered_request_count(sizes, 4, 16) == 16
assert 6 not in sizes
def test_dynamic_decode_capture_covers_schedule_gap_and_tail():
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=17,
num_speculative_tokens=4,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
num_speculative_tokens_per_batch_size=[(1, 2, 4), (5, 5, 1)],
)
VllmConfig._set_cudagraph_sizes(config)
sizes = compilation_config.cudagraph_capture_sizes
assert 20 in sizes
assert 34 in sizes
def test_default_cudagraph_capture_size_still_clamped_by_token_budget():
"""Decode coverage does not override the `max_num_batched_tokens` clamp.
A batch wider than the token budget cannot be scheduled in the first place,
so there is no decode step of that size to capture a graph for.
"""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=32,
num_speculative_tokens=16,
max_num_batched_tokens=512,
compilation_config=compilation_config,
)
VllmConfig._set_cudagraph_sizes(config)
# The token budget still clamps the final capture size.
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
decode_query_len = 17
token_grid_max = min(
32 * decode_query_len * 2,
default_max_graph_size,
)
widest_capturable_decode = min(
32 * decode_query_len,
512,
)
expected_max_size = max(token_grid_max, widest_capturable_decode)
assert compilation_config.max_cudagraph_capture_size == expected_max_size
assert (
compilation_config.max_cudagraph_capture_size
== compilation_config.cudagraph_capture_sizes[-1]
)
assert 544 not in compilation_config.cudagraph_capture_sizes
@pytest.mark.skipif(
not current_platform.support_static_graph_mode(),
reason="Skip if static graph mode is not supported",
+8 -10
View File
@@ -171,7 +171,7 @@ _DECODE_QUERY_LEN = 3
def _create_decode_vllm_config(
capture_sizes: list[int],
num_speculative_tokens: int = 0,
dynamic_spec_num_tokens: list[int] | None = None,
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
) -> MagicMock:
compilation_config = CompilationConfig(
cudagraph_mode="FULL_AND_PIECEWISE",
@@ -185,16 +185,12 @@ def _create_decode_vllm_config(
vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8)
vllm_config.parallel_config = ParallelConfig()
vllm_config.num_speculative_tokens = num_speculative_tokens
if dynamic_spec_num_tokens is None:
if dynamic_spec_schedule is None:
vllm_config.speculative_config = None
else:
speculative_config = MagicMock()
speculative_config.uses_dynamic_speculative_decoding.return_value = True
# Each entry is (range_start, range_end, num_speculative_tokens); only
# the third element is read by the manager.
speculative_config.num_speculative_tokens_per_batch_size = [
(0, 0, n) for n in dynamic_spec_num_tokens
]
speculative_config.num_speculative_tokens_per_batch_size = dynamic_spec_schedule
vllm_config.speculative_config = speculative_config
return vllm_config
@@ -204,7 +200,7 @@ def _make_spec_decode_manager(
decode_query_len: int = _DECODE_QUERY_LEN,
capture_sizes: list[int] | None = None,
num_speculative_tokens: int = 0,
dynamic_spec_num_tokens: list[int] | None = None,
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
@@ -220,7 +216,7 @@ def _make_spec_decode_manager(
vllm_config=_create_decode_vllm_config(
capture_sizes or [1, 2, 4, 8, 16, 24],
num_speculative_tokens=num_speculative_tokens,
dynamic_spec_num_tokens=dynamic_spec_num_tokens,
dynamic_spec_schedule=dynamic_spec_schedule,
),
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
@@ -343,7 +339,9 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
decode_query_len=4,
capture_sizes=[2, 4, 6, 8],
num_speculative_tokens=3,
dynamic_spec_num_tokens=[0, 1, 2, 3], # -> decode_query_lens [1, 2, 3, 4]
# max_num_seqs is 8, and K narrows as the batch grows, so the schedule
# covers K 3, 2, 1 and 0, giving decode_query_lens [1, 2, 3, 4].
dynamic_spec_schedule=[(1, 2, 3), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
)
full_descs = manager._capture_descs[CUDAGraphMode.FULL]
+35 -4
View File
@@ -151,6 +151,37 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
assert desc.num_active_loras == 0
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
"""Clamp configured K=5 to K=3, yielding query length 4 rather than 6."""
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(
gpu_cudagraph_utils.current_platform,
"get_global_graph_pool",
lambda: None,
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=16,
max_spec_tokens=3,
num_spec_per_batch_size=[(1, 16, 5)],
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=4,
)
full_query_lens = {
desc.uniform_token_count for desc in manager._capture_descs[CUDAGraphMode.FULL]
}
assert full_query_lens == {4}
def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
"""DSD should use PIECEWISE when the batch is not a uniform decode batch.
@@ -331,7 +362,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
"""DSD should only capture FULL graphs for query lengths in the schedule.
With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the
With a partial schedule of ``(1, 31, 4)`` and ``(32, 128, 3)``, only the
scheduled speculative-token counts (K = 4 and K = 3) become decode query
lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get
FULL graphs, while every other query length (e.g. the lower values 1, 2, 3)
@@ -343,9 +374,9 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
max_decode_query_len = max_spec_tokens + 1
# (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are
# scheduled, so FULL decode graphs should exist for query lengths K + 1,
# i.e. exactly {5, 4}.
num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)]
# scheduled in non-overlapping tiers, so FULL decode graphs should exist
# for query lengths K + 1, i.e. exactly {5, 4}.
num_spec_per_batch_size = [(1, 31, 4), (32, 128, 3)]
scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size}
monkeypatch.setattr(
+3 -1
View File
@@ -703,7 +703,9 @@ class CompilationConfig:
If not specified, max_cudagraph_capture_size is capped at 512 by default,
or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory
scenarios with small max_num_seqs, and limits capture of large graphs that
increase startup time and memory usage.
increase startup time and memory usage. When the uniform decode query length
exceeds one, uniform decode sizes are appended and can raise the final value
above this default.
"""
dynamic_shapes_config: DynamicShapesConfig = field(
+122 -6
View File
@@ -618,6 +618,27 @@ class VllmConfig:
return self.num_speculative_tokens
return 0
@property
def uniform_decode_query_len(self) -> int:
"""Query length of every request in a uniform decode batch.
A decode step submits one query for the newly sampled token plus one
for each draft token, so the widest uniform decode batch the scheduler
can build is `max_num_seqs * uniform_decode_query_len` tokens. Anything
that has to cover a decode batch reads this, so the sizing rule cannot
drift between the places that apply it.
This deliberately does not derive from the KV slots a drafter reserves
past the target's query range, which is a *reservation* contract rather
than a query-length one. The two do not differ by a constant: DFlash
reserves `num_speculative_tokens + 1` slots yet still verifies `1 +
num_speculative_tokens` queries, while EAGLE reserves
`num_speculative_tokens` and verifies the same `1 + n`. Deriving one
from the other would under-size EAGLE by a full request width, which is
the failure this property exists to prevent.
"""
return 1 + self.num_speculative_tokens
@property
def use_v2_model_runner(self) -> bool:
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
@@ -1897,8 +1918,10 @@ class VllmConfig:
```python
default_max_graph_size = 1024 if is_data_center_blackwell else 512
max_graph_size = min(max_num_seqs * decode_query_len * 2,
default_max_graph_size)
decode_query_len = self.uniform_decode_query_len
max_graph_size = min(
max_num_seqs * decode_query_len * 2, default_max_graph_size
)
# 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
# up to max_graph_size
cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
@@ -1906,7 +1929,10 @@ class VllmConfig:
`max_num_batched_tokens` is also appended to the list if it fits
within `max_cudagraph_capture_size`, so the max batch size is captured
even when off-stride.
even when off-stride. Likewise, when the uniform decode query length
exceeds one, the widest uniform decode batch (`max_num_seqs *
decode_query_len`) is appended when it fits, since it need not land on
an 8- or 16-token stride.
In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`
will be the final sizes to capture cudagraph (in ascending order).
@@ -1945,17 +1971,98 @@ class VllmConfig:
max_cudagraph_capture_size = (
self.compilation_config.max_cudagraph_capture_size
)
# Decode sizes to cover, in tokens. Populated only when a request
# is more than one token wide and only when the default is computed
# here, so an explicit capture range is left exactly as configured.
uniform_decode_sizes: list[int] = []
if max_cudagraph_capture_size is None:
from vllm.platforms import current_platform
decode_query_len = 1 + self.num_speculative_tokens
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
decode_query_len = self.uniform_decode_query_len
max_num_seqs = self.scheduler_config.max_num_seqs
max_cudagraph_capture_size = min(
self.scheduler_config.max_num_seqs * decode_query_len * 2,
default_max_graph_size,
max_num_seqs * decode_query_len * 2, default_max_graph_size
)
if decode_query_len > 1:
# A uniform decode batch is decode_query_len tokens per
# request, so the widest one is far outside this ceiling.
# Coverage comes from appending the decode sizes rather than
# extending the token-strided grid. Extending that grid to
# the widest decode size produces 581 sizes at
# max_num_seqs=512 and 16 draft tokens, versus 100 with the
# request-count grid.
#
# The grid would not buy decode coverage anyway. Dispatch
# requires an exact multiple of decode_query_len, so a
# token-strided entry is only usable when it happens to be
# one; at query length 17 a captured 560 rounds to 561 and
# is rejected. Scaling a request-count grid keeps every
# entry usable and the count comparable to the non-
# speculative case.
def request_counts(max_reqs: int) -> list[int]:
# At most the platform default number of requests,
# mirroring the one-token-per-request decode ceiling.
max_reqs = min(max_reqs, default_max_graph_size)
counts = [n for n in (1, 2, 4) if n <= max_reqs]
counts += list(range(8, min(max_reqs + 1, 256), 8))
counts += list(range(256, max_reqs + 1, 16))
return sorted(set(counts + [max_reqs]))
# Dynamic speculative decoding picks the draft width from
# the batch size, so a decode step is only uniform within a
# tier and each tier needs its own sizes. Scaling by the
# widest one alone leaves the narrower tiers short: the
# manager rounds a capture size up to a multiple of the
# tier's query length and drops it once the implied request
# count exceeds max_num_seqs, so at query length 3 sizes
# built from 17 stop covering at 227 of 256 requests.
decode_tiers = [(decode_query_len, max_num_seqs)]
speculative_config = self.speculative_config
if (
speculative_config is not None
and speculative_config.uses_dynamic_speculative_decoding()
):
from vllm.v1.spec_decode.dynamic.utils import (
build_dynamic_sd_schedule_lookup,
)
schedule = (
speculative_config.num_speculative_tokens_per_batch_size
)
assert schedule is not None
# Read the tiers off the dense lookup the scheduler
# runs on, so the clamp against num_speculative_tokens
# and the carry-forward through gaps and the tail
# cannot drift from it. Validation lives elsewhere; an
# invalid schedule keeps the single-tier default.
try:
dense_schedule = build_dynamic_sd_schedule_lookup(
schedule,
vllm_max_batch_size=max_num_seqs,
vllm_num_speculative_tokens=self.num_speculative_tokens,
)
except ValueError:
pass
else:
# Ascending batch size, so the last write per
# query length is the widest batch running at it.
widest_batch: dict[int, int] = {}
for batch_size, num_spec in enumerate(
dense_schedule[1:], start=1
):
widest_batch[num_spec + 1] = batch_size
decode_tiers = list(widest_batch.items())
uniform_decode_sizes = sorted(
{
n * query_len
for query_len, tier_max_reqs in decode_tiers
for n in request_counts(tier_max_reqs)
}
)
max_num_tokens = self.scheduler_config.max_num_batched_tokens
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)
@@ -2003,6 +2110,13 @@ class VllmConfig:
and max_num_tokens not in cudagraph_capture_sizes
):
cudagraph_capture_sizes.append(max_num_tokens)
# These extend past the token-strided ceiling, which counts one
# token per request. valid_max_size below raises the final
# max_cudagraph_capture_size to the widest of them. They remain
# filtered by max_num_tokens.
cudagraph_capture_sizes += [
size for size in uniform_decode_sizes if size <= max_num_tokens
]
# de-duplicate and sort the sizes
cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
@@ -2010,6 +2124,8 @@ class VllmConfig:
self.parallel_config.tensor_parallel_size > 1
and self.compilation_config.pass_config.enable_sp
):
# Sequence parallelism only captures TP-divisible sizes, so a
# wider non-divisible decode batch cannot be captured under SP.
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
cudagraph_capture_sizes
)
+12 -9
View File
@@ -34,6 +34,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
@@ -209,21 +210,23 @@ class CudaGraphManager:
speculative_config
and speculative_config.uses_dynamic_speculative_decoding()
):
num_spec_per_batch_size = (
speculative_config.num_speculative_tokens_per_batch_size
)
# uses_dynamic_speculative_decoding() guarantees this is set.
assert num_spec_per_batch_size is not None
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
# _per_step. Recover num_new_sampled_tokens_per_step
# from the values the manager already has.
num_new_sampled_tokens_per_step = (
self.decode_query_len - self.vllm_config.num_speculative_tokens
)
# Each entry is (range_start, range_end, num_speculative_tokens).
decode_query_lens = [
x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size
]
dense_schedule = build_dynamic_sd_schedule_lookup(
speculative_config.num_speculative_tokens_per_batch_size,
vllm_max_batch_size=self.max_num_reqs,
vllm_num_speculative_tokens=self.vllm_config.num_speculative_tokens,
)
decode_query_lens = sorted(
{
num_spec + num_new_sampled_tokens_per_step
for num_spec in dense_schedule[1:]
}
)
else:
decode_query_lens = [self.decode_query_len]