[Misc] Remove pad_for_cudagraphs from config (#30143)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
86c69dc54c
commit
2261340806
@@ -2,14 +2,20 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.fix_functionalization import FixFunctionalizationPass
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode, ParallelConfig, VllmConfig
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CUDAGraphMode,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.compilation import CompilationMode, PassConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -17,6 +23,7 @@ from vllm.utils.torch_utils import (
|
||||
_is_torch_equal_or_newer,
|
||||
is_torch_equal,
|
||||
)
|
||||
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
|
||||
|
||||
# This import automatically registers `torch.ops.silly.attention`
|
||||
from . import silly_attention # noqa: F401
|
||||
@@ -472,6 +479,19 @@ def test_cached_compilation_config(default_vllm_config):
|
||||
assert "torch.ops._C.static_scaled_fp8_quant.default(" in code
|
||||
|
||||
|
||||
def _create_vllm_config_for_validation(
|
||||
compilation_config: CompilationConfig,
|
||||
) -> MagicMock:
|
||||
"""Helper to create a mock VllmConfig for padding validation testing."""
|
||||
mock_config = MagicMock(spec=VllmConfig)
|
||||
mock_config.compilation_config = compilation_config
|
||||
mock_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8)
|
||||
mock_config.parallel_config = ParallelConfig()
|
||||
mock_config.speculative_config = None
|
||||
mock_config.lora_config = None
|
||||
return mock_config
|
||||
|
||||
|
||||
def test_compile_sizes_padding_validation():
|
||||
"""Test that compile_sizes with values that would be padded raises an error."""
|
||||
# cudagraph_capture_sizes=[1, 2, 4, 8] means:
|
||||
@@ -488,29 +508,39 @@ def test_compile_sizes_padding_validation():
|
||||
cudagraph_capture_sizes=[1, 2, 4, 8],
|
||||
max_cudagraph_capture_size=8,
|
||||
compile_sizes=[3],
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
)
|
||||
config.post_init_cudagraph_sizes()
|
||||
dispatcher = CudagraphDispatcher(_create_vllm_config_for_validation(config))
|
||||
dispatcher.initialize_cudagraph_keys(CUDAGraphMode.FULL)
|
||||
|
||||
with pytest.raises(ValueError, match="would be padded to"):
|
||||
config = CompilationConfig(
|
||||
cudagraph_capture_sizes=[1, 2, 4, 8],
|
||||
max_cudagraph_capture_size=8,
|
||||
compile_sizes=[5],
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
)
|
||||
config.post_init_cudagraph_sizes()
|
||||
dispatcher = CudagraphDispatcher(_create_vllm_config_for_validation(config))
|
||||
dispatcher.initialize_cudagraph_keys(CUDAGraphMode.FULL)
|
||||
|
||||
config = CompilationConfig(
|
||||
cudagraph_capture_sizes=[1, 2, 4, 8],
|
||||
max_cudagraph_capture_size=8,
|
||||
compile_sizes=[1, 2, 4, 8],
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
)
|
||||
config.post_init_cudagraph_sizes()
|
||||
assert sorted(config.compile_sizes) == [1, 2, 4, 8]
|
||||
dispatcher = CudagraphDispatcher(_create_vllm_config_for_validation(config))
|
||||
dispatcher.initialize_cudagraph_keys(CUDAGraphMode.FULL) # Should not raise
|
||||
|
||||
config = CompilationConfig(
|
||||
cudagraph_capture_sizes=[1, 2, 4, 8],
|
||||
max_cudagraph_capture_size=8,
|
||||
compile_sizes=["cudagraph_capture_sizes"],
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
)
|
||||
config.post_init_cudagraph_sizes()
|
||||
assert sorted(config.compile_sizes) == [1, 2, 4, 8]
|
||||
@@ -535,3 +565,5 @@ def test_compile_sizes_padding_validation():
|
||||
)
|
||||
config.post_init_cudagraph_sizes()
|
||||
assert sorted(config.compile_sizes) == [3, 5, 7]
|
||||
dispatcher = CudagraphDispatcher(_create_vllm_config_for_validation(config))
|
||||
dispatcher.initialize_cudagraph_keys(CUDAGraphMode.NONE) # Should not raise
|
||||
|
||||
@@ -9,6 +9,7 @@ from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
|
||||
|
||||
from ...utils import check_logprobs_close, check_outputs_equal
|
||||
|
||||
@@ -172,7 +173,14 @@ def test_mamba_cache_cg_padding(
|
||||
tensor dimensions aren't compatible.
|
||||
"""
|
||||
vllm_config = EngineArgs(model=model, trust_remote_code=True).create_engine_config()
|
||||
while len(example_prompts) == vllm_config.pad_for_cudagraph(len(example_prompts)):
|
||||
cudagraph_dispatcher = CudagraphDispatcher(vllm_config)
|
||||
cudagraph_dispatcher.initialize_cudagraph_keys(
|
||||
vllm_config.compilation_config.cudagraph_mode
|
||||
)
|
||||
while (
|
||||
len(example_prompts)
|
||||
== cudagraph_dispatcher.dispatch(len(example_prompts))[1].num_tokens
|
||||
):
|
||||
example_prompts.append(example_prompts[0])
|
||||
|
||||
try:
|
||||
|
||||
@@ -61,9 +61,6 @@ def _create_vllm_config(
|
||||
)
|
||||
|
||||
compilation_config.post_init_cudagraph_sizes()
|
||||
mock_config.pad_for_cudagraph = (
|
||||
lambda batch_size: compilation_config.bs_to_padded_graph_size[batch_size]
|
||||
)
|
||||
|
||||
return mock_config
|
||||
|
||||
@@ -169,6 +166,7 @@ class TestCudagraphDispatcher:
|
||||
rt_mode, key = dispatcher.dispatch(
|
||||
num_tokens=8, uniform_decode=False, has_lora=False, disable_full=True
|
||||
)
|
||||
|
||||
if "PIECEWISE" in cudagraph_mode_str: # string contains check
|
||||
assert rt_mode == CUDAGraphMode.PIECEWISE
|
||||
assert key == desc_full_exact.relax_for_mixed_batch_cudagraphs()
|
||||
@@ -360,7 +358,7 @@ class TestCudagraphIntegration:
|
||||
):
|
||||
full_wrapper(input_1)
|
||||
|
||||
rt_mode, key = self.dispatcher.dispatch(desc_1)
|
||||
rt_mode, key = self.dispatcher.dispatch(num_tokens=desc_1.num_tokens)
|
||||
# 1. Capture first shape
|
||||
action = self._run_and_monitor_call(full_wrapper, input_1, rt_mode, key)
|
||||
assert action == "capture_global"
|
||||
@@ -369,7 +367,7 @@ class TestCudagraphIntegration:
|
||||
action = self._run_and_monitor_call(full_wrapper, input_1, rt_mode, key)
|
||||
assert action == "replay"
|
||||
|
||||
rt_mode, key = self.dispatcher.dispatch(desc_2)
|
||||
rt_mode, key = self.dispatcher.dispatch(num_tokens=desc_2.num_tokens)
|
||||
# 3. Capture second shape
|
||||
action = self._run_and_monitor_call(full_wrapper, input_2, rt_mode, key)
|
||||
assert action == "capture_global"
|
||||
@@ -381,7 +379,7 @@ class TestCudagraphIntegration:
|
||||
assert action == "replay"
|
||||
|
||||
# 5. Bypass if no key match
|
||||
rt_mode, key = self.dispatcher.dispatch(desc_3_unseen)
|
||||
rt_mode, key = self.dispatcher.dispatch(num_tokens=desc_3_unseen.num_tokens)
|
||||
assert rt_mode == CUDAGraphMode.NONE
|
||||
action = self._run_and_monitor_call(full_wrapper, input_3, rt_mode, key)
|
||||
assert action == "bypass"
|
||||
|
||||
Reference in New Issue
Block a user