[MRV2][SD] Make Dynamic SD comatible with Full Cuda Graphs (#45953)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Benjamin Chislett
Nick Hill
parent
67ff0ae30f
commit
07516fda67
@@ -34,6 +34,7 @@ from vllm.config.vllm import (
|
||||
OptimizationLevel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
@@ -67,6 +68,36 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "expected_capture_sizes"),
|
||||
[
|
||||
(False, [4, 8, 12, 16]),
|
||||
(True, list(range(1, 17))),
|
||||
],
|
||||
)
|
||||
def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
||||
use_v2_model_runner,
|
||||
expected_capture_sizes,
|
||||
):
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
cudagraph_capture_sizes=list(range(1, 17)),
|
||||
)
|
||||
compilation_config.max_cudagraph_capture_size = 16
|
||||
compilation_config.post_init_cudagraph_sizes()
|
||||
|
||||
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
AttentionCGSupport.ALWAYS,
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=4,
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_config", "expected"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user