From 09c379ab6183085fd90e8195792cc63f5247cdf3 Mon Sep 17 00:00:00 2001 From: Andy Lo Date: Thu, 17 Sep 2026 19:09:18 +0100 Subject: [PATCH] [Bugfix] Max-load throughput cliff when `max_num_seqs` is not a multiple of 8 (#57355) Signed-off-by: Andy Lo Co-authored-by: OpenAI Codex --- tests/compile/test_config.py | 19 +++++++++++++++++++ vllm/config/vllm.py | 8 +++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 5c9132ccbf..7465ea442c 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -541,6 +541,25 @@ def _mock_config_for_cudagraph_sizes( return config +@pytest.mark.parametrize("max_num_seqs", [100, 101]) +def test_default_cudagraph_capture_sizes_cover_off_stride_max_num_seqs( + max_num_seqs: int, +) -> None: + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE + ) + config = _mock_config_for_cudagraph_sizes( + max_num_seqs=max_num_seqs, + num_speculative_tokens=0, + max_num_batched_tokens=32768, + compilation_config=compilation_config, + ) + + VllmConfig._set_cudagraph_sizes(config) + + assert max_num_seqs in compilation_config.cudagraph_capture_sizes + + @pytest.mark.parametrize( ("max_num_seqs", "num_speculative_tokens", "widest_is_captured"), [ diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index e5710dced3..4431743a3d 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2223,9 +2223,9 @@ class VllmConfig: max_cudagraph_capture_size = ( self.compilation_config.max_cudagraph_capture_size ) - # Decode sizes to cover, in tokens. Populated only when a request - # is more than one token wide and only when the default is computed - # here, so an explicit capture range is left exactly as configured. + # Decode sizes to cover, in tokens. Populated only when the default + # is computed here, so an explicit capture range is left exactly as + # configured. uniform_decode_sizes: list[int] = [] if max_cudagraph_capture_size is None: from vllm.platforms import current_platform @@ -2316,6 +2316,8 @@ class VllmConfig: if n * query_len <= max_cudagraph_capture_size } ) + elif max_num_seqs <= max_cudagraph_capture_size: + uniform_decode_sizes = [max_num_seqs] max_num_tokens = self.scheduler_config.max_num_batched_tokens max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)