[Perf] Raise Blackwell CUDA graph capture default to 1024 (#49390)
This commit is contained in:
@@ -492,6 +492,38 @@ def test_cudagraph_sizes_post_init(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.support_static_graph_mode(),
|
||||
reason="Skip if static graph mode is not supported",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("is_blackwell", "expected_max_size"), [(False, 512), (True, 1024)]
|
||||
)
|
||||
def test_blackwell_cudagraph_default(is_blackwell, expected_max_size):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = MagicMock(enforce_eager=False)
|
||||
vllm_config.scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=512,
|
||||
max_num_batched_tokens=2048,
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
)
|
||||
vllm_config.compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=is_blackwell,
|
||||
):
|
||||
vllm_config._set_cudagraph_sizes()
|
||||
|
||||
assert (
|
||||
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.support_static_graph_mode(),
|
||||
reason="Skip if not cudagraph mode supported",
|
||||
|
||||
@@ -700,10 +700,10 @@ class CompilationConfig:
|
||||
[1, 2, 4] + list(range(8, 256, 8)) + list(
|
||||
range(256, max_cudagraph_capture_size + 1, 16))
|
||||
|
||||
If not specified, max_cudagraph_capture_size is set to min(max_num_seqs*2,
|
||||
512) by default. This voids OOM in tight memory scenarios with small
|
||||
max_num_seqs, and prevents capture of many large graphs (>512) that would
|
||||
greatly increase startup time with limited performance benefit.
|
||||
If not specified, max_cudagraph_capture_size is capped at 512 by default,
|
||||
or 1024 on data center Blackwell GPUs. This avoids OOM in tight memory
|
||||
scenarios with small max_num_seqs, and limits capture of large graphs that
|
||||
increase startup time and memory usage.
|
||||
"""
|
||||
|
||||
dynamic_shapes_config: DynamicShapesConfig = field(
|
||||
|
||||
+12
-4
@@ -1808,7 +1808,9 @@ class VllmConfig:
|
||||
capture as:
|
||||
|
||||
```python
|
||||
max_graph_size = min(max_num_seqs * 2, 512)
|
||||
default_max_graph_size = 1024 if is_data_center_blackwell else 512
|
||||
max_graph_size = min(max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size)
|
||||
# 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
|
||||
# up to max_graph_size
|
||||
cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
|
||||
@@ -1832,8 +1834,8 @@ class VllmConfig:
|
||||
Example:
|
||||
With `max_num_batched_tokens = 8192`, and typical sequences
|
||||
averaging ~32 tokens, most practical batch sizes fall below 256.
|
||||
However, the system will still allow capture sizes up to 512 if
|
||||
shape and memory permit.
|
||||
However, the system will still allow capture sizes up to the
|
||||
platform default if shape and memory permit.
|
||||
|
||||
Note:
|
||||
If users explicitly specify cudagraph capture sizes in the
|
||||
@@ -1856,9 +1858,15 @@ class VllmConfig:
|
||||
self.compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
if max_cudagraph_capture_size is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
decode_query_len = 1 + self.num_speculative_tokens
|
||||
default_max_graph_size = (
|
||||
1024 if current_platform.is_device_capability_family(100) else 512
|
||||
)
|
||||
max_cudagraph_capture_size = min(
|
||||
self.scheduler_config.max_num_seqs * decode_query_len * 2, 512
|
||||
self.scheduler_config.max_num_seqs * decode_query_len * 2,
|
||||
default_max_graph_size,
|
||||
)
|
||||
max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)
|
||||
|
||||
Reference in New Issue
Block a user