[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
+35
View File
@@ -347,6 +347,41 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
block-count guard must only fire for the real cache sizing."""
kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"):
compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
is_profiling=True,
)
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
@pytest.mark.parametrize(
("model_config", "expected"),
[