[Bugfix][MRV2] Dispatch uniform decode to a padded FULL cudagraph (#53407)

Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: xiaohuguo2023 <149615094+xiaohuguo2023@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
xiaohuguo2023
2026-08-25 06:59:38 +00:00
committed by GitHub
co-authored by Lucas Wilkinson Lucas Wilkinson
parent 4af586e185
commit c74dd2d208
2 changed files with 247 additions and 24 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -109,3 +110,232 @@ def test_full_capture_sets_graph_pool_id_before_cuda_graph(monkeypatch):
manager.capture(create_forward_fn)
mock_cuda_graph.assert_called_once()
_DECODE_QUERY_LEN = 3
def _create_decode_vllm_config(
capture_sizes: list[int],
num_speculative_tokens: int = 0,
dynamic_spec_num_tokens: list[int] | None = None,
) -> MagicMock:
compilation_config = CompilationConfig(
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_capture_sizes=capture_sizes,
)
compilation_config.max_cudagraph_capture_size = capture_sizes[-1]
compilation_config.post_init_cudagraph_sizes()
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.compilation_config = compilation_config
vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8)
vllm_config.parallel_config = ParallelConfig()
vllm_config.num_speculative_tokens = num_speculative_tokens
if dynamic_spec_num_tokens is None:
vllm_config.speculative_config = None
else:
speculative_config = MagicMock()
speculative_config.uses_dynamic_speculative_decoding.return_value = True
# Each entry is (range_start, range_end, num_speculative_tokens); only
# the third element is read by the manager.
speculative_config.num_speculative_tokens_per_batch_size = [
(0, 0, n) for n in dynamic_spec_num_tokens
]
vllm_config.speculative_config = speculative_config
return vllm_config
def _make_spec_decode_manager(
monkeypatch,
decode_query_len: int = _DECODE_QUERY_LEN,
capture_sizes: list[int] | None = None,
num_speculative_tokens: int = 0,
dynamic_spec_num_tokens: list[int] | None = None,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(
gpu_cudagraph_utils.current_platform,
"get_global_graph_pool",
lambda: object(),
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=_create_decode_vllm_config(
capture_sizes or [1, 2, 4, 8, 16, 24],
num_speculative_tokens=num_speculative_tokens,
dynamic_spec_num_tokens=dynamic_spec_num_tokens,
),
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=decode_query_len,
)
manager._graphs_captured = True
return manager
def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
manager = _make_spec_decode_manager(monkeypatch)
assert [
(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[(12, 0)]
] == [
(CUDAGraphMode.FULL, 18),
(CUDAGraphMode.PIECEWISE, 16),
]
desc = manager.dispatch(
num_reqs=4,
num_tokens=12,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.uniform_token_count == _DECODE_QUERY_LEN
assert desc.num_tokens == 18
assert desc.num_reqs == 6
def test_uniform_decode_exact_match_is_not_over_padded(monkeypatch):
manager = _make_spec_decode_manager(monkeypatch)
desc = manager.dispatch(
num_reqs=3,
num_tokens=9,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.num_tokens == 9
assert desc.num_reqs == 3
def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
manager = _make_spec_decode_manager(monkeypatch)
desc = manager.dispatch(
num_reqs=2,
num_tokens=12,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_tokens == 16
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
"""A mixed batch must not fall to eager where only decode graphs are staged.
``round_up(size, decode_query_len)`` lands FULL decode graphs on token counts
the PIECEWISE ladder never uses -- with capture sizes [1, 2, 4, 8, 16, 24]
and query length 3, FULL covers 3, 6, 9 and 18 while PIECEWISE has only
1, 2, 4, 8, 16, 24. Building each mode's candidate ranges independently keeps
a PIECEWISE graph reachable there. Deriving the ranges from the staged sizes
instead offers a mixed batch nothing but decode descriptors, whose
``uniform_token_count`` it can never match, so it runs fully eager.
"""
manager = _make_spec_decode_manager(monkeypatch)
decode_only_token_counts = sorted(
{desc.num_tokens for desc in manager._capture_descs[CUDAGraphMode.FULL]}
- {desc.num_tokens for desc in manager._capture_descs[CUDAGraphMode.PIECEWISE]}
)
# Guard the premise: with no such token counts this test would cover nothing.
assert decode_only_token_counts
for num_tokens in decode_only_token_counts:
desc = manager.dispatch(
num_reqs=1,
num_tokens=num_tokens,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE, num_tokens
assert desc.uniform_token_count is None, num_tokens
assert desc.num_tokens >= num_tokens, num_tokens
def test_uniform_decode_beyond_capture_ladder_falls_back(monkeypatch):
manager = _make_spec_decode_manager(monkeypatch)
desc = manager.dispatch(
num_reqs=9,
num_tokens=27,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.NONE
def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
"""Every captured decode graph must remain reachable from dispatch().
Under dynamic speculative decoding ``decode_query_lens`` is a list, and the
staging loop rounds each capture size up to *every* query length. Several
decode graphs therefore land on the same ``num_tokens`` -- e.g. capture size
2 stages both ``round_up(2, 1) == 2`` and ``round_up(2, 2) == 2``. Building
the candidate ranges one descriptor at a time would give all but the first
of those an empty range, so they would never enter a candidate list and
their batches would silently fall through to PIECEWISE.
"""
manager = _make_spec_decode_manager(
monkeypatch,
decode_query_len=4,
capture_sizes=[2, 4, 6, 8],
num_speculative_tokens=3,
dynamic_spec_num_tokens=[0, 1, 2, 3], # -> decode_query_lens [1, 2, 3, 4]
)
full_descs = manager._capture_descs[CUDAGraphMode.FULL]
by_num_tokens: dict[int, list] = defaultdict(list)
for desc in full_descs:
by_num_tokens[desc.num_tokens].append(desc)
# Guard the premise: without collisions this test would not cover the bug.
assert any(len(descs) > 1 for descs in by_num_tokens.values())
for desc in full_descs:
assert desc in manager._candidates[(desc.num_tokens, 0)], desc
assert (
manager.dispatch(
num_reqs=desc.num_reqs,
num_tokens=desc.num_tokens,
uniform_token_count=desc.uniform_token_count,
num_active_loras=0,
)
== desc
), desc
@pytest.mark.parametrize(
"decode_query_len,capture_sizes",
[(1, [1, 2, 4, 8]), (2, [2, 4, 8, 16]), (8, [8, 16, 32, 64])],
)
def test_divisor_query_len_dispatch_is_unchanged(
monkeypatch, decode_query_len, capture_sizes
):
manager = _make_spec_decode_manager(
monkeypatch,
decode_query_len=decode_query_len,
capture_sizes=capture_sizes,
)
max_reqs = capture_sizes[-1] // decode_query_len
for num_reqs in range(1, max_reqs + 1):
num_tokens = num_reqs * decode_query_len
desc = manager.dispatch(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=decode_query_len,
num_active_loras=0,
)
expected = min(s for s in capture_sizes if s >= num_tokens)
assert desc.cg_mode == CUDAGraphMode.FULL, num_tokens
assert desc.num_tokens == expected, num_tokens
assert desc.num_reqs == expected // decode_query_len, num_tokens
+17 -24
View File
@@ -4,7 +4,7 @@ import gc
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from itertools import product
from itertools import groupby, product
from typing import TYPE_CHECKING, Any, NamedTuple, Protocol
import torch
@@ -194,9 +194,6 @@ class CudaGraphManager:
separate_decode_routine = self.cudagraph_mode.separate_routine()
max_cg_capture_size = self.compilation_config.max_cudagraph_capture_size
descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = (
defaultdict(list)
)
descs_by_mode: defaultdict[CUDAGraphMode, list[BatchExecutionDescriptor]] = (
defaultdict(list)
)
@@ -244,7 +241,6 @@ class CudaGraphManager:
num_active_loras=num_active_loras,
)
descs_by_mode[decode_mode].append(desc)
descs_by_token_lora[(num_tokens, num_active_loras)].append(desc)
# Capture uniform decode specfifc graphs if required
# (i.e. separate decode routine)
elif separate_decode_routine and decode_mode and not self.varlen_decode:
@@ -270,9 +266,6 @@ class CudaGraphManager:
# avoid duplicate graphs
if desc not in descs_by_mode[decode_mode]:
descs_by_mode[decode_mode].append(desc)
descs_by_token_lora[
(rounded_num_tokens, num_active_loras)
].append(desc)
if mixed_mode:
# for PIECEWISE graphs there is no limit on requests when replaying
@@ -290,27 +283,27 @@ class CudaGraphManager:
num_active_loras=num_active_loras,
)
descs_by_mode[mixed_mode].append(desc)
descs_by_token_lora[(num_tokens, num_active_loras)].append(desc)
if not descs_by_token_lora:
return
all_token_counts = sorted({k[0] for k in descs_by_token_lora})
current_range_start = 0
for token_cg_size in all_token_counts:
for i in range(current_range_start, token_cg_size + 1):
for num_active_loras in self.lora_capture_cases:
staging_key = (token_cg_size, num_active_loras)
if staging_key in descs_by_token_lora:
self._candidates[(i, num_active_loras)] = descs_by_token_lora[
staging_key
]
current_range_start = token_cg_size + 1
for mode, descs in descs_by_mode.items():
descs.sort(key=lambda d: d.num_tokens, reverse=True)
self._capture_descs[mode] = descs
for mode in (CUDAGraphMode.FULL, CUDAGraphMode.PIECEWISE):
mode_descs = tuple(reversed(descs_by_mode.get(mode, [])))
for num_active_loras in self.lora_capture_cases:
lora_descs = [
d for d in mode_descs if d.num_active_loras == num_active_loras
]
current_range_start = 0
# Dynamic speculative decoding can produce multiple graphs with the same
# num_tokens. Group them so each graph covers the same candidate range.
for num_tokens, group in groupby(lora_descs, lambda d: d.num_tokens):
matching = list(group)
for i in range(current_range_start, num_tokens + 1):
key = (i, num_active_loras)
self._candidates.setdefault(key, []).extend(matching)
current_range_start = num_tokens + 1
def needs_capture(self) -> bool:
return len(self._capture_descs) > 0