[Bugfix][Spec Decode] Capture the widest uniform decode batch by default (#50488)

Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com>
Signed-off-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Signed-off-by: Rahul Chalamala <rahul@modal.com>
Co-authored-by: Janelle Cai <janelle.cai@modal.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Devin <devin@cognition.ai>
This commit is contained in:
Rahul Chalamala
2026-08-28 22:38:13 -07:00
committed by GitHub
co-authored by Janelle Cai Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Devin
parent a758a9f671
commit d3d79ffc1e
6 changed files with 621 additions and 30 deletions
+8 -10
View File
@@ -171,7 +171,7 @@ _DECODE_QUERY_LEN = 3
def _create_decode_vllm_config(
capture_sizes: list[int],
num_speculative_tokens: int = 0,
dynamic_spec_num_tokens: list[int] | None = None,
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
) -> MagicMock:
compilation_config = CompilationConfig(
cudagraph_mode="FULL_AND_PIECEWISE",
@@ -185,16 +185,12 @@ def _create_decode_vllm_config(
vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8)
vllm_config.parallel_config = ParallelConfig()
vllm_config.num_speculative_tokens = num_speculative_tokens
if dynamic_spec_num_tokens is None:
if dynamic_spec_schedule is None:
vllm_config.speculative_config = None
else:
speculative_config = MagicMock()
speculative_config.uses_dynamic_speculative_decoding.return_value = True
# Each entry is (range_start, range_end, num_speculative_tokens); only
# the third element is read by the manager.
speculative_config.num_speculative_tokens_per_batch_size = [
(0, 0, n) for n in dynamic_spec_num_tokens
]
speculative_config.num_speculative_tokens_per_batch_size = dynamic_spec_schedule
vllm_config.speculative_config = speculative_config
return vllm_config
@@ -204,7 +200,7 @@ def _make_spec_decode_manager(
decode_query_len: int = _DECODE_QUERY_LEN,
capture_sizes: list[int] | None = None,
num_speculative_tokens: int = 0,
dynamic_spec_num_tokens: list[int] | None = None,
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
@@ -220,7 +216,7 @@ def _make_spec_decode_manager(
vllm_config=_create_decode_vllm_config(
capture_sizes or [1, 2, 4, 8, 16, 24],
num_speculative_tokens=num_speculative_tokens,
dynamic_spec_num_tokens=dynamic_spec_num_tokens,
dynamic_spec_schedule=dynamic_spec_schedule,
),
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
@@ -343,7 +339,9 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
decode_query_len=4,
capture_sizes=[2, 4, 6, 8],
num_speculative_tokens=3,
dynamic_spec_num_tokens=[0, 1, 2, 3], # -> decode_query_lens [1, 2, 3, 4]
# max_num_seqs is 8, and K narrows as the batch grows, so the schedule
# covers K 3, 2, 1 and 0, giving decode_query_lens [1, 2, 3, 4].
dynamic_spec_schedule=[(1, 2, 3), (3, 4, 2), (5, 6, 1), (7, 8, 0)],
)
full_descs = manager._capture_descs[CUDAGraphMode.FULL]
+35 -4
View File
@@ -151,6 +151,37 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
assert desc.num_active_loras == 0
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
"""Clamp configured K=5 to K=3, yielding query length 4 rather than 6."""
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(
gpu_cudagraph_utils.current_platform,
"get_global_graph_pool",
lambda: None,
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=16,
max_spec_tokens=3,
num_spec_per_batch_size=[(1, 16, 5)],
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=4,
)
full_query_lens = {
desc.uniform_token_count for desc in manager._capture_descs[CUDAGraphMode.FULL]
}
assert full_query_lens == {4}
def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
"""DSD should use PIECEWISE when the batch is not a uniform decode batch.
@@ -331,7 +362,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
"""DSD should only capture FULL graphs for query lengths in the schedule.
With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the
With a partial schedule of ``(1, 31, 4)`` and ``(32, 128, 3)``, only the
scheduled speculative-token counts (K = 4 and K = 3) become decode query
lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get
FULL graphs, while every other query length (e.g. the lower values 1, 2, 3)
@@ -343,9 +374,9 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
max_decode_query_len = max_spec_tokens + 1
# (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are
# scheduled, so FULL decode graphs should exist for query lengths K + 1,
# i.e. exactly {5, 4}.
num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)]
# scheduled in non-overlapping tiers, so FULL decode graphs should exist
# for query lengths K + 1, i.e. exactly {5, 4}.
num_spec_per_batch_size = [(1, 31, 4), (32, 128, 3)]
scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size}
monkeypatch.setattr(