[Bugfix][Spec Decode] Capture the widest uniform decode batch by default (#50488)
Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Signed-off-by: Rahul Chalamala <rahul@modal.com> Co-authored-by: Janelle Cai <janelle.cai@modal.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Devin <devin@cognition.ai>
This commit is contained in:
co-authored by
Janelle Cai
Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Devin
parent
a758a9f671
commit
d3d79ffc1e
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -15,13 +16,16 @@ from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CUDAGraphMode,
|
||||
ModelConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.compilation import CompilationMode, PassConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
_is_torch_equal_or_newer,
|
||||
is_torch_equal,
|
||||
@@ -497,6 +501,443 @@ def test_cudagraph_sizes_post_init(
|
||||
)
|
||||
|
||||
|
||||
def _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs: int,
|
||||
num_speculative_tokens: int,
|
||||
max_num_batched_tokens: int,
|
||||
compilation_config: CompilationConfig,
|
||||
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Mock VllmConfig wired up enough to run `_set_cudagraph_sizes`.
|
||||
|
||||
`num_speculative_tokens` and `uniform_decode_query_len` are filled in by
|
||||
calling the real property functions, so the tests below cover the
|
||||
derivation from `speculative_config` and not just the arithmetic
|
||||
downstream of it.
|
||||
"""
|
||||
config = MagicMock(spec=VllmConfig)
|
||||
config.compilation_config = compilation_config
|
||||
config.scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
)
|
||||
config.parallel_config = ParallelConfig()
|
||||
config.model_config = MagicMock()
|
||||
config.model_config.enforce_eager = False
|
||||
config.performance_mode = None
|
||||
config.diffusion_config = None
|
||||
schedule = num_speculative_tokens_per_batch_size
|
||||
config.speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
num_speculative_tokens_per_batch_size=schedule,
|
||||
uses_dynamic_speculative_decoding=lambda: schedule is not None,
|
||||
)
|
||||
if num_speculative_tokens
|
||||
else None
|
||||
)
|
||||
config.num_speculative_tokens = VllmConfig.num_speculative_tokens.fget(config)
|
||||
config.uniform_decode_query_len = VllmConfig.uniform_decode_query_len.fget(config)
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("max_num_seqs", "num_speculative_tokens"),
|
||||
[
|
||||
# No speculation: the 2x headroom under the platform ceiling, unchanged.
|
||||
(8, 0),
|
||||
(32, 0),
|
||||
# Speculating, but the widest decode batch still fits under the ceiling.
|
||||
(64, 7),
|
||||
(256, 1),
|
||||
# Widest decode batch above the ceiling, which must not cut below it.
|
||||
(32, 16),
|
||||
(64, 16),
|
||||
# Widest decode batch off the capture stride, above the ceiling.
|
||||
(33, 16),
|
||||
# ... and off the stride while below it, where the ceiling stands but
|
||||
# the generated sizes would otherwise stop at 400.
|
||||
(24, 16),
|
||||
],
|
||||
)
|
||||
def test_default_cudagraph_capture_size_covers_widest_uniform_decode(
|
||||
max_num_seqs, num_speculative_tokens
|
||||
):
|
||||
"""The widest uniform decode batch has to be inside the captured range.
|
||||
|
||||
A decode step presents up to `max_num_seqs * (1 + num_speculative_tokens)`
|
||||
tokens. If that size is not captured the decode path silently falls back to
|
||||
eager while the resolved mode still reports full decode graphs, so neither
|
||||
the platform's token-denominated ceiling nor the capture stride may stop
|
||||
short of it.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
decode_query_len = 1 + num_speculative_tokens
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
token_grid_max = min(
|
||||
max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
widest_uniform_decode = max_num_seqs * decode_query_len
|
||||
expected_max_size = max(token_grid_max, widest_uniform_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert widest_uniform_decode in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_sizes_keep_token_grid_bounded():
|
||||
"""Keep the token grid bounded; only request counts may exceed the default.
|
||||
|
||||
This guards the measured 581 versus 100 capture-size regression at
|
||||
max_num_seqs=512 with 16 draft tokens.
|
||||
"""
|
||||
max_num_seqs = 512
|
||||
decode_query_len = 17
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=max_num_seqs,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
token_grid_max = min(max_num_seqs * decode_query_len * 2, default_max_graph_size)
|
||||
token_grid = [size for size in [1, 2, 4] if size <= token_grid_max]
|
||||
token_grid += list(range(8, min(token_grid_max + 1, 256), 8))
|
||||
token_grid += list(range(256, token_grid_max + 1, 16))
|
||||
|
||||
max_request_count = min(max_num_seqs, default_max_graph_size)
|
||||
request_counts = [count for count in [1, 2, 4] if count <= max_request_count]
|
||||
request_counts += list(range(8, min(max_request_count + 1, 256), 8))
|
||||
request_counts += list(range(256, max_request_count + 1, 16))
|
||||
request_counts.append(max_request_count)
|
||||
expected_sizes = sorted(
|
||||
set(token_grid + [count * decode_query_len for count in request_counts])
|
||||
)
|
||||
|
||||
assert compilation_config.cudagraph_capture_sizes == expected_sizes
|
||||
assert all(
|
||||
size <= default_max_graph_size or size % decode_query_len == 0
|
||||
for size in compilation_config.cudagraph_capture_sizes
|
||||
)
|
||||
|
||||
|
||||
def test_cudagraph_capture_sizes_respect_sequence_parallelism():
|
||||
"""Sequence-parallel capture sizes stay divisible by tensor parallel size."""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
compilation_config.pass_config.enable_sp = True
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=32,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
config.parallel_config = SimpleNamespace(tensor_parallel_size=2)
|
||||
config.update_sizes_for_sequence_parallelism = lambda sizes: (
|
||||
VllmConfig.update_sizes_for_sequence_parallelism(config, sizes)
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
assert all(size % 2 == 0 for size in compilation_config.cudagraph_capture_sizes)
|
||||
assert 544 in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_real_vllm_config_captures_widest_ngram_decode_batch():
|
||||
"""Real config post-init preserves the widest static ngram decode batch."""
|
||||
model_config = ModelConfig(model="facebook/opt-125m")
|
||||
speculative_config = SpeculativeConfig(
|
||||
prompt_lookup_min=1,
|
||||
prompt_lookup_max=1,
|
||||
num_speculative_tokens=16,
|
||||
method="ngram",
|
||||
)
|
||||
scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=32,
|
||||
max_num_batched_tokens=32768,
|
||||
)
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
config = VllmConfig(
|
||||
model_config=model_config,
|
||||
speculative_config=speculative_config,
|
||||
scheduler_config=scheduler_config,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
assert 544 in config.compilation_config.cudagraph_capture_sizes
|
||||
assert (
|
||||
config.compilation_config.max_cudagraph_capture_size
|
||||
== config.compilation_config.cudagraph_capture_sizes[-1]
|
||||
== 544
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_num_seqs", [8, 32, 256, 300, 512, 600, 1024, 2048])
|
||||
def test_default_cudagraph_capture_size_unchanged_without_speculation(max_num_seqs):
|
||||
"""Without speculation the default must reproduce the historical formula.
|
||||
|
||||
The platform ceiling bounds a request count, and without speculation a
|
||||
request is one token, so `min(max_num_seqs, ceiling) * 1` can never lift it.
|
||||
The result must match `min(max_num_seqs * 2, ceiling)` bit for bit.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=max_num_seqs,
|
||||
num_speculative_tokens=0,
|
||||
max_num_batched_tokens=1_000_000,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
assert compilation_config.max_cudagraph_capture_size == min(
|
||||
max_num_seqs * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_size_covers_a_single_speculative_token():
|
||||
"""One speculative token is already enough to lose the widest batch.
|
||||
|
||||
MTP at depth 1 gives a query length of 2, so 300 requests is a 600-token
|
||||
decode batch against the platform's token ceiling. Gating the unit conversion on a
|
||||
deeper speculative width would leave this common configuration dispatching
|
||||
its largest decode steps eager.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=300,
|
||||
num_speculative_tokens=1,
|
||||
max_num_batched_tokens=1_000_000,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
token_grid_max = min(300 * 2 * 2, default_max_graph_size)
|
||||
widest_uniform_decode = 300 * 2
|
||||
expected_max_size = max(token_grid_max, widest_uniform_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert 600 in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_size_caps_requests_not_tokens():
|
||||
"""The ceiling admits at most the platform default number of requests.
|
||||
|
||||
`max_num_seqs` far above the platform default must not push the capture
|
||||
range up without bound just because each request is now several tokens wide.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=1024,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=1_000_000,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
decode_query_len = 17
|
||||
token_grid_max = min(1024 * decode_query_len * 2, default_max_graph_size)
|
||||
widest_covered_decode = min(1024, default_max_graph_size) * decode_query_len
|
||||
expected_max_size = max(token_grid_max, widest_covered_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert expected_max_size == default_max_graph_size * 17
|
||||
|
||||
|
||||
def _widest_covered_request_count(capture_sizes, query_len, max_num_seqs):
|
||||
"""Widest request count `CudaGraphManager` can build a decode graph for.
|
||||
|
||||
Mirrors `_init_candidates`: a capture size is rounded up to a multiple of
|
||||
the tier's query length, then dropped once the implied request count runs
|
||||
past `max_num_seqs`.
|
||||
"""
|
||||
reachable = [
|
||||
cdiv(size, query_len)
|
||||
for size in capture_sizes
|
||||
if cdiv(size, query_len) <= max_num_seqs
|
||||
]
|
||||
return max(reachable, default=0)
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_sizes_cover_every_dynamic_decode_width():
|
||||
"""Each scheduled draft width needs sizes over the range it applies to.
|
||||
|
||||
Dynamic speculative decoding picks the width from the batch size, so
|
||||
scaling by the widest one alone leaves the narrower tiers short: sizes
|
||||
built from query length 17 round up to multiples of 3 that imply more
|
||||
requests than the scheduler can run, and coverage stops at 227 of 256.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=256,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
# 16 draft tokens up to a batch of 16, then 2 out to 256.
|
||||
num_speculative_tokens_per_batch_size=[(1, 16, 16), (17, 256, 2)],
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
sizes = compilation_config.cudagraph_capture_sizes
|
||||
# The wide tier only ever runs to a batch of 16, the narrow one to 256.
|
||||
assert _widest_covered_request_count(sizes, 17, 256) >= 16
|
||||
assert _widest_covered_request_count(sizes, 3, 256) == 256
|
||||
|
||||
|
||||
def test_dynamic_decode_capture_clamps_configured_width():
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=16,
|
||||
num_speculative_tokens=3,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
num_speculative_tokens_per_batch_size=[(1, 16, 5)],
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
sizes = compilation_config.cudagraph_capture_sizes
|
||||
assert _widest_covered_request_count(sizes, 4, 16) == 16
|
||||
assert 6 not in sizes
|
||||
|
||||
|
||||
def test_dynamic_decode_capture_covers_schedule_gap_and_tail():
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=17,
|
||||
num_speculative_tokens=4,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
num_speculative_tokens_per_batch_size=[(1, 2, 4), (5, 5, 1)],
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
sizes = compilation_config.cudagraph_capture_sizes
|
||||
assert 20 in sizes
|
||||
assert 34 in sizes
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_size_still_clamped_by_token_budget():
|
||||
"""Decode coverage does not override the `max_num_batched_tokens` clamp.
|
||||
|
||||
A batch wider than the token budget cannot be scheduled in the first place,
|
||||
so there is no decode step of that size to capture a graph for.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=32,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=512,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
# The token budget still clamps the final capture size.
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
decode_query_len = 17
|
||||
token_grid_max = min(
|
||||
32 * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
widest_capturable_decode = min(
|
||||
32 * decode_query_len,
|
||||
512,
|
||||
)
|
||||
expected_max_size = max(token_grid_max, widest_capturable_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert 544 not in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.support_static_graph_mode(),
|
||||
reason="Skip if static graph mode is not supported",
|
||||
|
||||
@@ -171,7 +171,7 @@ _DECODE_QUERY_LEN = 3
|
||||
def _create_decode_vllm_config(
|
||||
capture_sizes: list[int],
|
||||
num_speculative_tokens: int = 0,
|
||||
dynamic_spec_num_tokens: list[int] | None = None,
|
||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
@@ -185,16 +185,12 @@ def _create_decode_vllm_config(
|
||||
vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8)
|
||||
vllm_config.parallel_config = ParallelConfig()
|
||||
vllm_config.num_speculative_tokens = num_speculative_tokens
|
||||
if dynamic_spec_num_tokens is None:
|
||||
if dynamic_spec_schedule is None:
|
||||
vllm_config.speculative_config = None
|
||||
else:
|
||||
speculative_config = MagicMock()
|
||||
speculative_config.uses_dynamic_speculative_decoding.return_value = True
|
||||
# Each entry is (range_start, range_end, num_speculative_tokens); only
|
||||
# the third element is read by the manager.
|
||||
speculative_config.num_speculative_tokens_per_batch_size = [
|
||||
(0, 0, n) for n in dynamic_spec_num_tokens
|
||||
]
|
||||
speculative_config.num_speculative_tokens_per_batch_size = dynamic_spec_schedule
|
||||
vllm_config.speculative_config = speculative_config
|
||||
return vllm_config
|
||||
|
||||
@@ -204,7 +200,7 @@ def _make_spec_decode_manager(
|
||||
decode_query_len: int = _DECODE_QUERY_LEN,
|
||||
capture_sizes: list[int] | None = None,
|
||||
num_speculative_tokens: int = 0,
|
||||
dynamic_spec_num_tokens: list[int] | None = None,
|
||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||
) -> gpu_cudagraph_utils.CudaGraphManager:
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
@@ -220,7 +216,7 @@ def _make_spec_decode_manager(
|
||||
vllm_config=_create_decode_vllm_config(
|
||||
capture_sizes or [1, 2, 4, 8, 16, 24],
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
dynamic_spec_num_tokens=dynamic_spec_num_tokens,
|
||||
dynamic_spec_schedule=dynamic_spec_schedule,
|
||||
),
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
@@ -343,7 +339,9 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
|
||||
decode_query_len=4,
|
||||
capture_sizes=[2, 4, 6, 8],
|
||||
num_speculative_tokens=3,
|
||||
dynamic_spec_num_tokens=[0, 1, 2, 3], # -> decode_query_lens [1, 2, 3, 4]
|
||||
# max_num_seqs is 8, and K narrows as the batch grows, so the schedule
|
||||
# covers K 3, 2, 1 and 0, giving decode_query_lens [1, 2, 3, 4].
|
||||
dynamic_spec_schedule=[(1, 2, 3), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
|
||||
)
|
||||
|
||||
full_descs = manager._capture_descs[CUDAGraphMode.FULL]
|
||||
|
||||
@@ -151,6 +151,37 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
|
||||
"""Clamp configured K=5 to K=3, yielding query length 4 rather than 6."""
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils.current_platform,
|
||||
"get_global_graph_pool",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=16,
|
||||
max_spec_tokens=3,
|
||||
num_spec_per_batch_size=[(1, 16, 5)],
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=4,
|
||||
)
|
||||
|
||||
full_query_lens = {
|
||||
desc.uniform_token_count for desc in manager._capture_descs[CUDAGraphMode.FULL]
|
||||
}
|
||||
assert full_query_lens == {4}
|
||||
|
||||
|
||||
def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
"""DSD should use PIECEWISE when the batch is not a uniform decode batch.
|
||||
|
||||
@@ -331,7 +362,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
"""DSD should only capture FULL graphs for query lengths in the schedule.
|
||||
|
||||
With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the
|
||||
With a partial schedule of ``(1, 31, 4)`` and ``(32, 128, 3)``, only the
|
||||
scheduled speculative-token counts (K = 4 and K = 3) become decode query
|
||||
lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get
|
||||
FULL graphs, while every other query length (e.g. the lower values 1, 2, 3)
|
||||
@@ -343,9 +374,9 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
max_decode_query_len = max_spec_tokens + 1
|
||||
|
||||
# (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are
|
||||
# scheduled, so FULL decode graphs should exist for query lengths K + 1,
|
||||
# i.e. exactly {5, 4}.
|
||||
num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)]
|
||||
# scheduled in non-overlapping tiers, so FULL decode graphs should exist
|
||||
# for query lengths K + 1, i.e. exactly {5, 4}.
|
||||
num_spec_per_batch_size = [(1, 31, 4), (32, 128, 3)]
|
||||
scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size}
|
||||
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -703,7 +703,9 @@ class CompilationConfig:
|
||||
If not specified, max_cudagraph_capture_size is capped at 512 by default,
|
||||
or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory
|
||||
scenarios with small max_num_seqs, and limits capture of large graphs that
|
||||
increase startup time and memory usage.
|
||||
increase startup time and memory usage. When the uniform decode query length
|
||||
exceeds one, uniform decode sizes are appended and can raise the final value
|
||||
above this default.
|
||||
"""
|
||||
|
||||
dynamic_shapes_config: DynamicShapesConfig = field(
|
||||
|
||||
+122
-6
@@ -618,6 +618,27 @@ class VllmConfig:
|
||||
return self.num_speculative_tokens
|
||||
return 0
|
||||
|
||||
@property
|
||||
def uniform_decode_query_len(self) -> int:
|
||||
"""Query length of every request in a uniform decode batch.
|
||||
|
||||
A decode step submits one query for the newly sampled token plus one
|
||||
for each draft token, so the widest uniform decode batch the scheduler
|
||||
can build is `max_num_seqs * uniform_decode_query_len` tokens. Anything
|
||||
that has to cover a decode batch reads this, so the sizing rule cannot
|
||||
drift between the places that apply it.
|
||||
|
||||
This deliberately does not derive from the KV slots a drafter reserves
|
||||
past the target's query range, which is a *reservation* contract rather
|
||||
than a query-length one. The two do not differ by a constant: DFlash
|
||||
reserves `num_speculative_tokens + 1` slots yet still verifies `1 +
|
||||
num_speculative_tokens` queries, while EAGLE reserves
|
||||
`num_speculative_tokens` and verifies the same `1 + n`. Deriving one
|
||||
from the other would under-size EAGLE by a full request width, which is
|
||||
the failure this property exists to prevent.
|
||||
"""
|
||||
return 1 + self.num_speculative_tokens
|
||||
|
||||
@property
|
||||
def use_v2_model_runner(self) -> bool:
|
||||
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
|
||||
@@ -1897,8 +1918,10 @@ class VllmConfig:
|
||||
|
||||
```python
|
||||
default_max_graph_size = 1024 if is_data_center_blackwell else 512
|
||||
max_graph_size = min(max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size)
|
||||
decode_query_len = self.uniform_decode_query_len
|
||||
max_graph_size = min(
|
||||
max_num_seqs * decode_query_len * 2, default_max_graph_size
|
||||
)
|
||||
# 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
|
||||
# up to max_graph_size
|
||||
cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
|
||||
@@ -1906,7 +1929,10 @@ class VllmConfig:
|
||||
|
||||
`max_num_batched_tokens` is also appended to the list if it fits
|
||||
within `max_cudagraph_capture_size`, so the max batch size is captured
|
||||
even when off-stride.
|
||||
even when off-stride. Likewise, when the uniform decode query length
|
||||
exceeds one, the widest uniform decode batch (`max_num_seqs *
|
||||
decode_query_len`) is appended when it fits, since it need not land on
|
||||
an 8- or 16-token stride.
|
||||
|
||||
In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`
|
||||
will be the final sizes to capture cudagraph (in ascending order).
|
||||
@@ -1945,17 +1971,98 @@ class VllmConfig:
|
||||
max_cudagraph_capture_size = (
|
||||
self.compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
# Decode sizes to cover, in tokens. Populated only when a request
|
||||
# is more than one token wide and only when the default is computed
|
||||
# here, so an explicit capture range is left exactly as configured.
|
||||
uniform_decode_sizes: list[int] = []
|
||||
if max_cudagraph_capture_size is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
decode_query_len = 1 + self.num_speculative_tokens
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
decode_query_len = self.uniform_decode_query_len
|
||||
max_num_seqs = self.scheduler_config.max_num_seqs
|
||||
max_cudagraph_capture_size = min(
|
||||
self.scheduler_config.max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
max_num_seqs * decode_query_len * 2, default_max_graph_size
|
||||
)
|
||||
if decode_query_len > 1:
|
||||
# A uniform decode batch is decode_query_len tokens per
|
||||
# request, so the widest one is far outside this ceiling.
|
||||
# Coverage comes from appending the decode sizes rather than
|
||||
# extending the token-strided grid. Extending that grid to
|
||||
# the widest decode size produces 581 sizes at
|
||||
# max_num_seqs=512 and 16 draft tokens, versus 100 with the
|
||||
# request-count grid.
|
||||
#
|
||||
# The grid would not buy decode coverage anyway. Dispatch
|
||||
# requires an exact multiple of decode_query_len, so a
|
||||
# token-strided entry is only usable when it happens to be
|
||||
# one; at query length 17 a captured 560 rounds to 561 and
|
||||
# is rejected. Scaling a request-count grid keeps every
|
||||
# entry usable and the count comparable to the non-
|
||||
# speculative case.
|
||||
def request_counts(max_reqs: int) -> list[int]:
|
||||
# At most the platform default number of requests,
|
||||
# mirroring the one-token-per-request decode ceiling.
|
||||
max_reqs = min(max_reqs, default_max_graph_size)
|
||||
counts = [n for n in (1, 2, 4) if n <= max_reqs]
|
||||
counts += list(range(8, min(max_reqs + 1, 256), 8))
|
||||
counts += list(range(256, max_reqs + 1, 16))
|
||||
return sorted(set(counts + [max_reqs]))
|
||||
|
||||
# Dynamic speculative decoding picks the draft width from
|
||||
# the batch size, so a decode step is only uniform within a
|
||||
# tier and each tier needs its own sizes. Scaling by the
|
||||
# widest one alone leaves the narrower tiers short: the
|
||||
# manager rounds a capture size up to a multiple of the
|
||||
# tier's query length and drops it once the implied request
|
||||
# count exceeds max_num_seqs, so at query length 3 sizes
|
||||
# built from 17 stop covering at 227 of 256 requests.
|
||||
decode_tiers = [(decode_query_len, max_num_seqs)]
|
||||
speculative_config = self.speculative_config
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.uses_dynamic_speculative_decoding()
|
||||
):
|
||||
from vllm.v1.spec_decode.dynamic.utils import (
|
||||
build_dynamic_sd_schedule_lookup,
|
||||
)
|
||||
|
||||
schedule = (
|
||||
speculative_config.num_speculative_tokens_per_batch_size
|
||||
)
|
||||
assert schedule is not None
|
||||
# Read the tiers off the dense lookup the scheduler
|
||||
# runs on, so the clamp against num_speculative_tokens
|
||||
# and the carry-forward through gaps and the tail
|
||||
# cannot drift from it. Validation lives elsewhere; an
|
||||
# invalid schedule keeps the single-tier default.
|
||||
try:
|
||||
dense_schedule = build_dynamic_sd_schedule_lookup(
|
||||
schedule,
|
||||
vllm_max_batch_size=max_num_seqs,
|
||||
vllm_num_speculative_tokens=self.num_speculative_tokens,
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
# Ascending batch size, so the last write per
|
||||
# query length is the widest batch running at it.
|
||||
widest_batch: dict[int, int] = {}
|
||||
for batch_size, num_spec in enumerate(
|
||||
dense_schedule[1:], start=1
|
||||
):
|
||||
widest_batch[num_spec + 1] = batch_size
|
||||
decode_tiers = list(widest_batch.items())
|
||||
|
||||
uniform_decode_sizes = sorted(
|
||||
{
|
||||
n * query_len
|
||||
for query_len, tier_max_reqs in decode_tiers
|
||||
for n in request_counts(tier_max_reqs)
|
||||
}
|
||||
)
|
||||
max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)
|
||||
|
||||
@@ -2003,6 +2110,13 @@ class VllmConfig:
|
||||
and max_num_tokens not in cudagraph_capture_sizes
|
||||
):
|
||||
cudagraph_capture_sizes.append(max_num_tokens)
|
||||
# These extend past the token-strided ceiling, which counts one
|
||||
# token per request. valid_max_size below raises the final
|
||||
# max_cudagraph_capture_size to the widest of them. They remain
|
||||
# filtered by max_num_tokens.
|
||||
cudagraph_capture_sizes += [
|
||||
size for size in uniform_decode_sizes if size <= max_num_tokens
|
||||
]
|
||||
# de-duplicate and sort the sizes
|
||||
cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
|
||||
|
||||
@@ -2010,6 +2124,8 @@ class VllmConfig:
|
||||
self.parallel_config.tensor_parallel_size > 1
|
||||
and self.compilation_config.pass_config.enable_sp
|
||||
):
|
||||
# Sequence parallelism only captures TP-divisible sizes, so a
|
||||
# wider non-divisible decode batch cannot be captured under SP.
|
||||
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
|
||||
cudagraph_capture_sizes
|
||||
)
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
@@ -209,21 +210,23 @@ class CudaGraphManager:
|
||||
speculative_config
|
||||
and speculative_config.uses_dynamic_speculative_decoding()
|
||||
):
|
||||
num_spec_per_batch_size = (
|
||||
speculative_config.num_speculative_tokens_per_batch_size
|
||||
)
|
||||
# uses_dynamic_speculative_decoding() guarantees this is set.
|
||||
assert num_spec_per_batch_size is not None
|
||||
# decode_query_len = num_speculative_steps + num_new_sampled_tokens
|
||||
# _per_step. Recover num_new_sampled_tokens_per_step
|
||||
# from the values the manager already has.
|
||||
num_new_sampled_tokens_per_step = (
|
||||
self.decode_query_len - self.vllm_config.num_speculative_tokens
|
||||
)
|
||||
# Each entry is (range_start, range_end, num_speculative_tokens).
|
||||
decode_query_lens = [
|
||||
x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size
|
||||
]
|
||||
dense_schedule = build_dynamic_sd_schedule_lookup(
|
||||
speculative_config.num_speculative_tokens_per_batch_size,
|
||||
vllm_max_batch_size=self.max_num_reqs,
|
||||
vllm_num_speculative_tokens=self.vllm_config.num_speculative_tokens,
|
||||
)
|
||||
decode_query_lens = sorted(
|
||||
{
|
||||
num_spec + num_new_sampled_tokens_per_step
|
||||
for num_spec in dense_schedule[1:]
|
||||
}
|
||||
)
|
||||
else:
|
||||
decode_query_lens = [self.decode_query_len]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user