[Bugfix][Spec Decode] Capture the widest uniform decode batch by default (#50488)
Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Signed-off-by: Rahul Chalamala <rahul@modal.com> Co-authored-by: Janelle Cai <janelle.cai@modal.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Devin <devin@cognition.ai>
This commit is contained in:
co-authored by
Janelle Cai
Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Devin
parent
a758a9f671
commit
d3d79ffc1e
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -15,13 +16,16 @@ from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CUDAGraphMode,
|
||||
ModelConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.compilation import CompilationMode, PassConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
_is_torch_equal_or_newer,
|
||||
is_torch_equal,
|
||||
@@ -497,6 +501,443 @@ def test_cudagraph_sizes_post_init(
|
||||
)
|
||||
|
||||
|
||||
def _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs: int,
|
||||
num_speculative_tokens: int,
|
||||
max_num_batched_tokens: int,
|
||||
compilation_config: CompilationConfig,
|
||||
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Mock VllmConfig wired up enough to run `_set_cudagraph_sizes`.
|
||||
|
||||
`num_speculative_tokens` and `uniform_decode_query_len` are filled in by
|
||||
calling the real property functions, so the tests below cover the
|
||||
derivation from `speculative_config` and not just the arithmetic
|
||||
downstream of it.
|
||||
"""
|
||||
config = MagicMock(spec=VllmConfig)
|
||||
config.compilation_config = compilation_config
|
||||
config.scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
)
|
||||
config.parallel_config = ParallelConfig()
|
||||
config.model_config = MagicMock()
|
||||
config.model_config.enforce_eager = False
|
||||
config.performance_mode = None
|
||||
config.diffusion_config = None
|
||||
schedule = num_speculative_tokens_per_batch_size
|
||||
config.speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
num_speculative_tokens_per_batch_size=schedule,
|
||||
uses_dynamic_speculative_decoding=lambda: schedule is not None,
|
||||
)
|
||||
if num_speculative_tokens
|
||||
else None
|
||||
)
|
||||
config.num_speculative_tokens = VllmConfig.num_speculative_tokens.fget(config)
|
||||
config.uniform_decode_query_len = VllmConfig.uniform_decode_query_len.fget(config)
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("max_num_seqs", "num_speculative_tokens"),
|
||||
[
|
||||
# No speculation: the 2x headroom under the platform ceiling, unchanged.
|
||||
(8, 0),
|
||||
(32, 0),
|
||||
# Speculating, but the widest decode batch still fits under the ceiling.
|
||||
(64, 7),
|
||||
(256, 1),
|
||||
# Widest decode batch above the ceiling, which must not cut below it.
|
||||
(32, 16),
|
||||
(64, 16),
|
||||
# Widest decode batch off the capture stride, above the ceiling.
|
||||
(33, 16),
|
||||
# ... and off the stride while below it, where the ceiling stands but
|
||||
# the generated sizes would otherwise stop at 400.
|
||||
(24, 16),
|
||||
],
|
||||
)
|
||||
def test_default_cudagraph_capture_size_covers_widest_uniform_decode(
|
||||
max_num_seqs, num_speculative_tokens
|
||||
):
|
||||
"""The widest uniform decode batch has to be inside the captured range.
|
||||
|
||||
A decode step presents up to `max_num_seqs * (1 + num_speculative_tokens)`
|
||||
tokens. If that size is not captured the decode path silently falls back to
|
||||
eager while the resolved mode still reports full decode graphs, so neither
|
||||
the platform's token-denominated ceiling nor the capture stride may stop
|
||||
short of it.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
decode_query_len = 1 + num_speculative_tokens
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
token_grid_max = min(
|
||||
max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
widest_uniform_decode = max_num_seqs * decode_query_len
|
||||
expected_max_size = max(token_grid_max, widest_uniform_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert widest_uniform_decode in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_sizes_keep_token_grid_bounded():
|
||||
"""Keep the token grid bounded; only request counts may exceed the default.
|
||||
|
||||
This guards the measured 581 versus 100 capture-size regression at
|
||||
max_num_seqs=512 with 16 draft tokens.
|
||||
"""
|
||||
max_num_seqs = 512
|
||||
decode_query_len = 17
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=max_num_seqs,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
token_grid_max = min(max_num_seqs * decode_query_len * 2, default_max_graph_size)
|
||||
token_grid = [size for size in [1, 2, 4] if size <= token_grid_max]
|
||||
token_grid += list(range(8, min(token_grid_max + 1, 256), 8))
|
||||
token_grid += list(range(256, token_grid_max + 1, 16))
|
||||
|
||||
max_request_count = min(max_num_seqs, default_max_graph_size)
|
||||
request_counts = [count for count in [1, 2, 4] if count <= max_request_count]
|
||||
request_counts += list(range(8, min(max_request_count + 1, 256), 8))
|
||||
request_counts += list(range(256, max_request_count + 1, 16))
|
||||
request_counts.append(max_request_count)
|
||||
expected_sizes = sorted(
|
||||
set(token_grid + [count * decode_query_len for count in request_counts])
|
||||
)
|
||||
|
||||
assert compilation_config.cudagraph_capture_sizes == expected_sizes
|
||||
assert all(
|
||||
size <= default_max_graph_size or size % decode_query_len == 0
|
||||
for size in compilation_config.cudagraph_capture_sizes
|
||||
)
|
||||
|
||||
|
||||
def test_cudagraph_capture_sizes_respect_sequence_parallelism():
|
||||
"""Sequence-parallel capture sizes stay divisible by tensor parallel size."""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
compilation_config.pass_config.enable_sp = True
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=32,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
config.parallel_config = SimpleNamespace(tensor_parallel_size=2)
|
||||
config.update_sizes_for_sequence_parallelism = lambda sizes: (
|
||||
VllmConfig.update_sizes_for_sequence_parallelism(config, sizes)
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
assert all(size % 2 == 0 for size in compilation_config.cudagraph_capture_sizes)
|
||||
assert 544 in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_real_vllm_config_captures_widest_ngram_decode_batch():
|
||||
"""Real config post-init preserves the widest static ngram decode batch."""
|
||||
model_config = ModelConfig(model="facebook/opt-125m")
|
||||
speculative_config = SpeculativeConfig(
|
||||
prompt_lookup_min=1,
|
||||
prompt_lookup_max=1,
|
||||
num_speculative_tokens=16,
|
||||
method="ngram",
|
||||
)
|
||||
scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=32,
|
||||
max_num_batched_tokens=32768,
|
||||
)
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
config = VllmConfig(
|
||||
model_config=model_config,
|
||||
speculative_config=speculative_config,
|
||||
scheduler_config=scheduler_config,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
assert 544 in config.compilation_config.cudagraph_capture_sizes
|
||||
assert (
|
||||
config.compilation_config.max_cudagraph_capture_size
|
||||
== config.compilation_config.cudagraph_capture_sizes[-1]
|
||||
== 544
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_num_seqs", [8, 32, 256, 300, 512, 600, 1024, 2048])
|
||||
def test_default_cudagraph_capture_size_unchanged_without_speculation(max_num_seqs):
|
||||
"""Without speculation the default must reproduce the historical formula.
|
||||
|
||||
The platform ceiling bounds a request count, and without speculation a
|
||||
request is one token, so `min(max_num_seqs, ceiling) * 1` can never lift it.
|
||||
The result must match `min(max_num_seqs * 2, ceiling)` bit for bit.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=max_num_seqs,
|
||||
num_speculative_tokens=0,
|
||||
max_num_batched_tokens=1_000_000,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
assert compilation_config.max_cudagraph_capture_size == min(
|
||||
max_num_seqs * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_size_covers_a_single_speculative_token():
|
||||
"""One speculative token is already enough to lose the widest batch.
|
||||
|
||||
MTP at depth 1 gives a query length of 2, so 300 requests is a 600-token
|
||||
decode batch against the platform's token ceiling. Gating the unit conversion on a
|
||||
deeper speculative width would leave this common configuration dispatching
|
||||
its largest decode steps eager.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=300,
|
||||
num_speculative_tokens=1,
|
||||
max_num_batched_tokens=1_000_000,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
token_grid_max = min(300 * 2 * 2, default_max_graph_size)
|
||||
widest_uniform_decode = 300 * 2
|
||||
expected_max_size = max(token_grid_max, widest_uniform_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert 600 in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_size_caps_requests_not_tokens():
|
||||
"""The ceiling admits at most the platform default number of requests.
|
||||
|
||||
`max_num_seqs` far above the platform default must not push the capture
|
||||
range up without bound just because each request is now several tokens wide.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=1024,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=1_000_000,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
decode_query_len = 17
|
||||
token_grid_max = min(1024 * decode_query_len * 2, default_max_graph_size)
|
||||
widest_covered_decode = min(1024, default_max_graph_size) * decode_query_len
|
||||
expected_max_size = max(token_grid_max, widest_covered_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert expected_max_size == default_max_graph_size * 17
|
||||
|
||||
|
||||
def _widest_covered_request_count(capture_sizes, query_len, max_num_seqs):
|
||||
"""Widest request count `CudaGraphManager` can build a decode graph for.
|
||||
|
||||
Mirrors `_init_candidates`: a capture size is rounded up to a multiple of
|
||||
the tier's query length, then dropped once the implied request count runs
|
||||
past `max_num_seqs`.
|
||||
"""
|
||||
reachable = [
|
||||
cdiv(size, query_len)
|
||||
for size in capture_sizes
|
||||
if cdiv(size, query_len) <= max_num_seqs
|
||||
]
|
||||
return max(reachable, default=0)
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_sizes_cover_every_dynamic_decode_width():
|
||||
"""Each scheduled draft width needs sizes over the range it applies to.
|
||||
|
||||
Dynamic speculative decoding picks the width from the batch size, so
|
||||
scaling by the widest one alone leaves the narrower tiers short: sizes
|
||||
built from query length 17 round up to multiples of 3 that imply more
|
||||
requests than the scheduler can run, and coverage stops at 227 of 256.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=256,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
# 16 draft tokens up to a batch of 16, then 2 out to 256.
|
||||
num_speculative_tokens_per_batch_size=[(1, 16, 16), (17, 256, 2)],
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
sizes = compilation_config.cudagraph_capture_sizes
|
||||
# The wide tier only ever runs to a batch of 16, the narrow one to 256.
|
||||
assert _widest_covered_request_count(sizes, 17, 256) >= 16
|
||||
assert _widest_covered_request_count(sizes, 3, 256) == 256
|
||||
|
||||
|
||||
def test_dynamic_decode_capture_clamps_configured_width():
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=16,
|
||||
num_speculative_tokens=3,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
num_speculative_tokens_per_batch_size=[(1, 16, 5)],
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
sizes = compilation_config.cudagraph_capture_sizes
|
||||
assert _widest_covered_request_count(sizes, 4, 16) == 16
|
||||
assert 6 not in sizes
|
||||
|
||||
|
||||
def test_dynamic_decode_capture_covers_schedule_gap_and_tail():
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=17,
|
||||
num_speculative_tokens=4,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
num_speculative_tokens_per_batch_size=[(1, 2, 4), (5, 5, 1)],
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
sizes = compilation_config.cudagraph_capture_sizes
|
||||
assert 20 in sizes
|
||||
assert 34 in sizes
|
||||
|
||||
|
||||
def test_default_cudagraph_capture_size_still_clamped_by_token_budget():
|
||||
"""Decode coverage does not override the `max_num_batched_tokens` clamp.
|
||||
|
||||
A batch wider than the token budget cannot be scheduled in the first place,
|
||||
so there is no decode step of that size to capture a graph for.
|
||||
"""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=32,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=512,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
# The token budget still clamps the final capture size.
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
decode_query_len = 17
|
||||
token_grid_max = min(
|
||||
32 * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
widest_capturable_decode = min(
|
||||
32 * decode_query_len,
|
||||
512,
|
||||
)
|
||||
expected_max_size = max(token_grid_max, widest_capturable_decode)
|
||||
assert compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
assert (
|
||||
compilation_config.max_cudagraph_capture_size
|
||||
== compilation_config.cudagraph_capture_sizes[-1]
|
||||
)
|
||||
assert 544 not in compilation_config.cudagraph_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.support_static_graph_mode(),
|
||||
reason="Skip if static graph mode is not supported",
|
||||
|
||||
@@ -171,7 +171,7 @@ _DECODE_QUERY_LEN = 3
|
||||
def _create_decode_vllm_config(
|
||||
capture_sizes: list[int],
|
||||
num_speculative_tokens: int = 0,
|
||||
dynamic_spec_num_tokens: list[int] | None = None,
|
||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
@@ -185,16 +185,12 @@ def _create_decode_vllm_config(
|
||||
vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8)
|
||||
vllm_config.parallel_config = ParallelConfig()
|
||||
vllm_config.num_speculative_tokens = num_speculative_tokens
|
||||
if dynamic_spec_num_tokens is None:
|
||||
if dynamic_spec_schedule is None:
|
||||
vllm_config.speculative_config = None
|
||||
else:
|
||||
speculative_config = MagicMock()
|
||||
speculative_config.uses_dynamic_speculative_decoding.return_value = True
|
||||
# Each entry is (range_start, range_end, num_speculative_tokens); only
|
||||
# the third element is read by the manager.
|
||||
speculative_config.num_speculative_tokens_per_batch_size = [
|
||||
(0, 0, n) for n in dynamic_spec_num_tokens
|
||||
]
|
||||
speculative_config.num_speculative_tokens_per_batch_size = dynamic_spec_schedule
|
||||
vllm_config.speculative_config = speculative_config
|
||||
return vllm_config
|
||||
|
||||
@@ -204,7 +200,7 @@ def _make_spec_decode_manager(
|
||||
decode_query_len: int = _DECODE_QUERY_LEN,
|
||||
capture_sizes: list[int] | None = None,
|
||||
num_speculative_tokens: int = 0,
|
||||
dynamic_spec_num_tokens: list[int] | None = None,
|
||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||
) -> gpu_cudagraph_utils.CudaGraphManager:
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
@@ -220,7 +216,7 @@ def _make_spec_decode_manager(
|
||||
vllm_config=_create_decode_vllm_config(
|
||||
capture_sizes or [1, 2, 4, 8, 16, 24],
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
dynamic_spec_num_tokens=dynamic_spec_num_tokens,
|
||||
dynamic_spec_schedule=dynamic_spec_schedule,
|
||||
),
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
@@ -343,7 +339,9 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
|
||||
decode_query_len=4,
|
||||
capture_sizes=[2, 4, 6, 8],
|
||||
num_speculative_tokens=3,
|
||||
dynamic_spec_num_tokens=[0, 1, 2, 3], # -> decode_query_lens [1, 2, 3, 4]
|
||||
# max_num_seqs is 8, and K narrows as the batch grows, so the schedule
|
||||
# covers K 3, 2, 1 and 0, giving decode_query_lens [1, 2, 3, 4].
|
||||
dynamic_spec_schedule=[(1, 2, 3), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
|
||||
)
|
||||
|
||||
full_descs = manager._capture_descs[CUDAGraphMode.FULL]
|
||||
|
||||
@@ -151,6 +151,37 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
|
||||
"""Clamp configured K=5 to K=3, yielding query length 4 rather than 6."""
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils.current_platform,
|
||||
"get_global_graph_pool",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=16,
|
||||
max_spec_tokens=3,
|
||||
num_spec_per_batch_size=[(1, 16, 5)],
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=4,
|
||||
)
|
||||
|
||||
full_query_lens = {
|
||||
desc.uniform_token_count for desc in manager._capture_descs[CUDAGraphMode.FULL]
|
||||
}
|
||||
assert full_query_lens == {4}
|
||||
|
||||
|
||||
def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
"""DSD should use PIECEWISE when the batch is not a uniform decode batch.
|
||||
|
||||
@@ -331,7 +362,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
"""DSD should only capture FULL graphs for query lengths in the schedule.
|
||||
|
||||
With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the
|
||||
With a partial schedule of ``(1, 31, 4)`` and ``(32, 128, 3)``, only the
|
||||
scheduled speculative-token counts (K = 4 and K = 3) become decode query
|
||||
lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get
|
||||
FULL graphs, while every other query length (e.g. the lower values 1, 2, 3)
|
||||
@@ -343,9 +374,9 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
max_decode_query_len = max_spec_tokens + 1
|
||||
|
||||
# (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are
|
||||
# scheduled, so FULL decode graphs should exist for query lengths K + 1,
|
||||
# i.e. exactly {5, 4}.
|
||||
num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)]
|
||||
# scheduled in non-overlapping tiers, so FULL decode graphs should exist
|
||||
# for query lengths K + 1, i.e. exactly {5, 4}.
|
||||
num_spec_per_batch_size = [(1, 31, 4), (32, 128, 3)]
|
||||
scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size}
|
||||
|
||||
monkeypatch.setattr(
|
||||
|
||||
Reference in New Issue
Block a user