From c74dd2d208389305e3a6ced992ee0632f939c0a0 Mon Sep 17 00:00:00 2001 From: xiaohuguo2023 <149615094+xiaohuguo2023@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:59:38 +0100 Subject: [PATCH] [Bugfix][MRV2] Dispatch uniform decode to a padded FULL cudagraph (#53407) Signed-off-by: Xiaohu Guo Signed-off-by: Lucas Wilkinson Signed-off-by: xiaohuguo2023 <149615094+xiaohuguo2023@users.noreply.github.com> Co-authored-by: Lucas Wilkinson Co-authored-by: Lucas Wilkinson --- tests/v1/cudagraph/test_cudagraph_manager.py | 230 +++++++++++++++++++ vllm/v1/worker/gpu/cudagraph_utils.py | 41 ++-- 2 files changed, 247 insertions(+), 24 deletions(-) diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 655fcf0d2e..b8cda3c11a 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections import defaultdict from contextlib import contextmanager from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -109,3 +110,232 @@ def test_full_capture_sets_graph_pool_id_before_cuda_graph(monkeypatch): manager.capture(create_forward_fn) mock_cuda_graph.assert_called_once() + + +_DECODE_QUERY_LEN = 3 + + +def _create_decode_vllm_config( + capture_sizes: list[int], + num_speculative_tokens: int = 0, + dynamic_spec_num_tokens: list[int] | None = None, +) -> MagicMock: + compilation_config = CompilationConfig( + cudagraph_mode="FULL_AND_PIECEWISE", + cudagraph_capture_sizes=capture_sizes, + ) + compilation_config.max_cudagraph_capture_size = capture_sizes[-1] + compilation_config.post_init_cudagraph_sizes() + + vllm_config = MagicMock(spec=VllmConfig) + vllm_config.compilation_config = compilation_config + vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8) + vllm_config.parallel_config = ParallelConfig() + vllm_config.num_speculative_tokens = num_speculative_tokens + if dynamic_spec_num_tokens is None: + vllm_config.speculative_config = None + else: + speculative_config = MagicMock() + speculative_config.uses_dynamic_speculative_decoding.return_value = True + # Each entry is (range_start, range_end, num_speculative_tokens); only + # the third element is read by the manager. + speculative_config.num_speculative_tokens_per_batch_size = [ + (0, 0, n) for n in dynamic_spec_num_tokens + ] + vllm_config.speculative_config = speculative_config + return vllm_config + + +def _make_spec_decode_manager( + monkeypatch, + decode_query_len: int = _DECODE_QUERY_LEN, + capture_sizes: list[int] | None = None, + num_speculative_tokens: int = 0, + dynamic_spec_num_tokens: list[int] | None = None, +) -> gpu_cudagraph_utils.CudaGraphManager: + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + monkeypatch.setattr( + gpu_cudagraph_utils.current_platform, + "get_global_graph_pool", + lambda: object(), + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=_create_decode_vllm_config( + capture_sizes or [1, 2, 4, 8, 16, 24], + num_speculative_tokens=num_speculative_tokens, + dynamic_spec_num_tokens=dynamic_spec_num_tokens, + ), + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=decode_query_len, + ) + manager._graphs_captured = True + return manager + + +def test_uniform_decode_pads_up_to_full_graph(monkeypatch): + manager = _make_spec_decode_manager(monkeypatch) + assert [ + (desc.cg_mode, desc.num_tokens) for desc in manager._candidates[(12, 0)] + ] == [ + (CUDAGraphMode.FULL, 18), + (CUDAGraphMode.PIECEWISE, 16), + ] + + desc = manager.dispatch( + num_reqs=4, + num_tokens=12, + uniform_token_count=_DECODE_QUERY_LEN, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.uniform_token_count == _DECODE_QUERY_LEN + assert desc.num_tokens == 18 + assert desc.num_reqs == 6 + + +def test_uniform_decode_exact_match_is_not_over_padded(monkeypatch): + manager = _make_spec_decode_manager(monkeypatch) + + desc = manager.dispatch( + num_reqs=3, + num_tokens=9, + uniform_token_count=_DECODE_QUERY_LEN, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.num_tokens == 9 + assert desc.num_reqs == 3 + + +def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch): + manager = _make_spec_decode_manager(monkeypatch) + + desc = manager.dispatch( + num_reqs=2, + num_tokens=12, + uniform_token_count=None, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_tokens == 16 + + +def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch): + """A mixed batch must not fall to eager where only decode graphs are staged. + + ``round_up(size, decode_query_len)`` lands FULL decode graphs on token counts + the PIECEWISE ladder never uses -- with capture sizes [1, 2, 4, 8, 16, 24] + and query length 3, FULL covers 3, 6, 9 and 18 while PIECEWISE has only + 1, 2, 4, 8, 16, 24. Building each mode's candidate ranges independently keeps + a PIECEWISE graph reachable there. Deriving the ranges from the staged sizes + instead offers a mixed batch nothing but decode descriptors, whose + ``uniform_token_count`` it can never match, so it runs fully eager. + """ + manager = _make_spec_decode_manager(monkeypatch) + + decode_only_token_counts = sorted( + {desc.num_tokens for desc in manager._capture_descs[CUDAGraphMode.FULL]} + - {desc.num_tokens for desc in manager._capture_descs[CUDAGraphMode.PIECEWISE]} + ) + # Guard the premise: with no such token counts this test would cover nothing. + assert decode_only_token_counts + + for num_tokens in decode_only_token_counts: + desc = manager.dispatch( + num_reqs=1, + num_tokens=num_tokens, + uniform_token_count=None, + num_active_loras=0, + ) + assert desc.cg_mode == CUDAGraphMode.PIECEWISE, num_tokens + assert desc.uniform_token_count is None, num_tokens + assert desc.num_tokens >= num_tokens, num_tokens + + +def test_uniform_decode_beyond_capture_ladder_falls_back(monkeypatch): + manager = _make_spec_decode_manager(monkeypatch) + + desc = manager.dispatch( + num_reqs=9, + num_tokens=27, + uniform_token_count=_DECODE_QUERY_LEN, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.NONE + + +def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch): + """Every captured decode graph must remain reachable from dispatch(). + + Under dynamic speculative decoding ``decode_query_lens`` is a list, and the + staging loop rounds each capture size up to *every* query length. Several + decode graphs therefore land on the same ``num_tokens`` -- e.g. capture size + 2 stages both ``round_up(2, 1) == 2`` and ``round_up(2, 2) == 2``. Building + the candidate ranges one descriptor at a time would give all but the first + of those an empty range, so they would never enter a candidate list and + their batches would silently fall through to PIECEWISE. + """ + manager = _make_spec_decode_manager( + monkeypatch, + decode_query_len=4, + capture_sizes=[2, 4, 6, 8], + num_speculative_tokens=3, + dynamic_spec_num_tokens=[0, 1, 2, 3], # -> decode_query_lens [1, 2, 3, 4] + ) + + full_descs = manager._capture_descs[CUDAGraphMode.FULL] + by_num_tokens: dict[int, list] = defaultdict(list) + for desc in full_descs: + by_num_tokens[desc.num_tokens].append(desc) + # Guard the premise: without collisions this test would not cover the bug. + assert any(len(descs) > 1 for descs in by_num_tokens.values()) + + for desc in full_descs: + assert desc in manager._candidates[(desc.num_tokens, 0)], desc + assert ( + manager.dispatch( + num_reqs=desc.num_reqs, + num_tokens=desc.num_tokens, + uniform_token_count=desc.uniform_token_count, + num_active_loras=0, + ) + == desc + ), desc + + +@pytest.mark.parametrize( + "decode_query_len,capture_sizes", + [(1, [1, 2, 4, 8]), (2, [2, 4, 8, 16]), (8, [8, 16, 32, 64])], +) +def test_divisor_query_len_dispatch_is_unchanged( + monkeypatch, decode_query_len, capture_sizes +): + manager = _make_spec_decode_manager( + monkeypatch, + decode_query_len=decode_query_len, + capture_sizes=capture_sizes, + ) + + max_reqs = capture_sizes[-1] // decode_query_len + for num_reqs in range(1, max_reqs + 1): + num_tokens = num_reqs * decode_query_len + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=decode_query_len, + num_active_loras=0, + ) + expected = min(s for s in capture_sizes if s >= num_tokens) + assert desc.cg_mode == CUDAGraphMode.FULL, num_tokens + assert desc.num_tokens == expected, num_tokens + assert desc.num_reqs == expected // decode_query_len, num_tokens diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index e09a1756a5..0bce6bd681 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -4,7 +4,7 @@ import gc from collections import defaultdict from collections.abc import Callable from dataclasses import dataclass -from itertools import product +from itertools import groupby, product from typing import TYPE_CHECKING, Any, NamedTuple, Protocol import torch @@ -194,9 +194,6 @@ class CudaGraphManager: separate_decode_routine = self.cudagraph_mode.separate_routine() max_cg_capture_size = self.compilation_config.max_cudagraph_capture_size - descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = ( - defaultdict(list) - ) descs_by_mode: defaultdict[CUDAGraphMode, list[BatchExecutionDescriptor]] = ( defaultdict(list) ) @@ -244,7 +241,6 @@ class CudaGraphManager: num_active_loras=num_active_loras, ) descs_by_mode[decode_mode].append(desc) - descs_by_token_lora[(num_tokens, num_active_loras)].append(desc) # Capture uniform decode specfifc graphs if required # (i.e. separate decode routine) elif separate_decode_routine and decode_mode and not self.varlen_decode: @@ -270,9 +266,6 @@ class CudaGraphManager: # avoid duplicate graphs if desc not in descs_by_mode[decode_mode]: descs_by_mode[decode_mode].append(desc) - descs_by_token_lora[ - (rounded_num_tokens, num_active_loras) - ].append(desc) if mixed_mode: # for PIECEWISE graphs there is no limit on requests when replaying @@ -290,27 +283,27 @@ class CudaGraphManager: num_active_loras=num_active_loras, ) descs_by_mode[mixed_mode].append(desc) - descs_by_token_lora[(num_tokens, num_active_loras)].append(desc) - - if not descs_by_token_lora: - return - - all_token_counts = sorted({k[0] for k in descs_by_token_lora}) - current_range_start = 0 - for token_cg_size in all_token_counts: - for i in range(current_range_start, token_cg_size + 1): - for num_active_loras in self.lora_capture_cases: - staging_key = (token_cg_size, num_active_loras) - if staging_key in descs_by_token_lora: - self._candidates[(i, num_active_loras)] = descs_by_token_lora[ - staging_key - ] - current_range_start = token_cg_size + 1 for mode, descs in descs_by_mode.items(): descs.sort(key=lambda d: d.num_tokens, reverse=True) self._capture_descs[mode] = descs + for mode in (CUDAGraphMode.FULL, CUDAGraphMode.PIECEWISE): + mode_descs = tuple(reversed(descs_by_mode.get(mode, []))) + for num_active_loras in self.lora_capture_cases: + lora_descs = [ + d for d in mode_descs if d.num_active_loras == num_active_loras + ] + current_range_start = 0 + # Dynamic speculative decoding can produce multiple graphs with the same + # num_tokens. Group them so each graph covers the same candidate range. + for num_tokens, group in groupby(lora_descs, lambda d: d.num_tokens): + matching = list(group) + for i in range(current_range_start, num_tokens + 1): + key = (i, num_active_loras) + self._candidates.setdefault(key, []).extend(matching) + current_range_start = num_tokens + 1 + def needs_capture(self) -> bool: return len(self._capture_descs) > 0