[Perf] Raise Blackwell CUDA graph capture default to 1024 (#49390)

This commit is contained in:
Lucas Wilkinson
2026-08-07 15:18:14 -04:00
committed by GitHub
parent 56a4b63d44
commit 021b7d985b
3 changed files with 48 additions and 8 deletions
+32
View File
@@ -492,6 +492,38 @@ def test_cudagraph_sizes_post_init(
)
@pytest.mark.skipif(
not current_platform.support_static_graph_mode(),
reason="Skip if static graph mode is not supported",
)
@pytest.mark.parametrize(
("is_blackwell", "expected_max_size"), [(False, 512), (True, 1024)]
)
def test_blackwell_cudagraph_default(is_blackwell, expected_max_size):
vllm_config = VllmConfig()
vllm_config.model_config = MagicMock(enforce_eager=False)
vllm_config.scheduler_config = SchedulerConfig(
max_num_seqs=512,
max_num_batched_tokens=2048,
max_model_len=2048,
is_encoder_decoder=False,
)
vllm_config.compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=is_blackwell,
):
vllm_config._set_cudagraph_sizes()
assert (
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
)
@pytest.mark.skipif(
not current_platform.support_static_graph_mode(),
reason="Skip if not cudagraph mode supported",
+4 -4
View File
@@ -700,10 +700,10 @@ class CompilationConfig:
[1, 2, 4] + list(range(8, 256, 8)) + list(
range(256, max_cudagraph_capture_size + 1, 16))
If not specified, max_cudagraph_capture_size is set to min(max_num_seqs*2,
512) by default. This voids OOM in tight memory scenarios with small
max_num_seqs, and prevents capture of many large graphs (>512) that would
greatly increase startup time with limited performance benefit.
If not specified, max_cudagraph_capture_size is capped at 512 by default,
or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory
scenarios with small max_num_seqs, and limits capture of large graphs that
increase startup time and memory usage.
"""
dynamic_shapes_config: DynamicShapesConfig = field(
+12 -4
View File
@@ -1808,7 +1808,9 @@ class VllmConfig:
capture as:
```python
max_graph_size = min(max_num_seqs * 2, 512)
default_max_graph_size = 1024 if is_data_center_blackwell else 512
max_graph_size = min(max_num_seqs * decode_query_len * 2,
default_max_graph_size)
# 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
# up to max_graph_size
cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
@@ -1832,8 +1834,8 @@ class VllmConfig:
Example:
With `max_num_batched_tokens = 8192`, and typical sequences
averaging ~32 tokens, most practical batch sizes fall below 256.
However, the system will still allow capture sizes up to 512 if
shape and memory permit.
However, the system will still allow capture sizes up to the
platform default if shape and memory permit.
Note:
If users explicitly specify cudagraph capture sizes in the
@@ -1856,9 +1858,15 @@ class VllmConfig:
self.compilation_config.max_cudagraph_capture_size
)
if max_cudagraph_capture_size is None:
from vllm.platforms import current_platform
decode_query_len = 1 + self.num_speculative_tokens
default_max_graph_size = (
1024 if current_platform.is_device_capability_family(100) else 512
)
max_cudagraph_capture_size = min(
self.scheduler_config.max_num_seqs * decode_query_len * 2, 512
self.scheduler_config.max_num_seqs * decode_query_len * 2,
default_max_graph_size,
)
max_num_tokens = self.scheduler_config.max_num_batched_tokens
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)