From 021b7d985b54264393b4c339f2823d783080942a Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 7 Aug 2026 15:18:14 -0400 Subject: [PATCH] [Perf] Raise Blackwell CUDA graph capture default to 1024 (#49390) --- tests/compile/test_config.py | 32 ++++++++++++++++++++++++++++++++ vllm/config/compilation.py | 8 ++++---- vllm/config/vllm.py | 16 ++++++++++++---- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index c45907ee6e..84a7733f18 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -492,6 +492,38 @@ def test_cudagraph_sizes_post_init( ) +@pytest.mark.skipif( + not current_platform.support_static_graph_mode(), + reason="Skip if static graph mode is not supported", +) +@pytest.mark.parametrize( + ("is_blackwell", "expected_max_size"), [(False, 512), (True, 1024)] +) +def test_blackwell_cudagraph_default(is_blackwell, expected_max_size): + vllm_config = VllmConfig() + vllm_config.model_config = MagicMock(enforce_eager=False) + vllm_config.scheduler_config = SchedulerConfig( + max_num_seqs=512, + max_num_batched_tokens=2048, + max_model_len=2048, + is_encoder_decoder=False, + ) + vllm_config.compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + ) + + with patch.object( + current_platform, + "is_device_capability_family", + return_value=is_blackwell, + ): + vllm_config._set_cudagraph_sizes() + + assert ( + vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size + ) + + @pytest.mark.skipif( not current_platform.support_static_graph_mode(), reason="Skip if not cudagraph mode supported", diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 4eb66613a8..0776a60a46 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -700,10 +700,10 @@ class CompilationConfig: [1, 2, 4] + list(range(8, 256, 8)) + list( range(256, max_cudagraph_capture_size + 1, 16)) - If not specified, max_cudagraph_capture_size is set to min(max_num_seqs*2, - 512) by default. This voids OOM in tight memory scenarios with small - max_num_seqs, and prevents capture of many large graphs (>512) that would - greatly increase startup time with limited performance benefit. + If not specified, max_cudagraph_capture_size is capped at 512 by default, + or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory + scenarios with small max_num_seqs, and limits capture of large graphs that + increase startup time and memory usage. """ dynamic_shapes_config: DynamicShapesConfig = field( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index edac3441c7..e4c8ea5289 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1808,7 +1808,9 @@ class VllmConfig: capture as: ```python - max_graph_size = min(max_num_seqs * 2, 512) + default_max_graph_size = 1024 if is_data_center_blackwell else 512 + max_graph_size = min(max_num_seqs * decode_query_len * 2, + default_max_graph_size) # 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16 # up to max_graph_size cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list( @@ -1832,8 +1834,8 @@ class VllmConfig: Example: With `max_num_batched_tokens = 8192`, and typical sequences averaging ~32 tokens, most practical batch sizes fall below 256. - However, the system will still allow capture sizes up to 512 if - shape and memory permit. + However, the system will still allow capture sizes up to the + platform default if shape and memory permit. Note: If users explicitly specify cudagraph capture sizes in the @@ -1856,9 +1858,15 @@ class VllmConfig: self.compilation_config.max_cudagraph_capture_size ) if max_cudagraph_capture_size is None: + from vllm.platforms import current_platform + decode_query_len = 1 + self.num_speculative_tokens + default_max_graph_size = ( + 1024 if current_platform.is_device_capability_family(100) else 512 + ) max_cudagraph_capture_size = min( - self.scheduler_config.max_num_seqs * decode_query_len * 2, 512 + self.scheduler_config.max_num_seqs * decode_query_len * 2, + default_max_graph_size, ) max_num_tokens = self.scheduler_config.max_num_batched_tokens max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)