[Model Runner V2] Reserve CUDA graph memory (#53306)
Signed-off-by: Anh Tran <anh.tran.3889@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Anh Tran <anh.tran.3889@gmail.com> Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
co-authored by
Anh Tran
anhtra3889
parent
8c2bbe00d5
commit
a4d70bef37
@@ -347,6 +347,41 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
||||
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||
|
||||
|
||||
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
|
||||
block-count guard must only fire for the real cache sizing."""
|
||||
kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4)
|
||||
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
)
|
||||
with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"):
|
||||
compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
AttentionCGSupport.ALWAYS,
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=1,
|
||||
use_v2_model_runner=True,
|
||||
tensor_parallel_size=1,
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_num_reqs=256,
|
||||
)
|
||||
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
)
|
||||
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
AttentionCGSupport.ALWAYS,
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=1,
|
||||
use_v2_model_runner=True,
|
||||
tensor_parallel_size=1,
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_num_reqs=256,
|
||||
is_profiling=True,
|
||||
)
|
||||
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_config", "expected"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user