[Frontend][torch.compile] CompilationConfig Overhaul (#20283): name change compilation level to compilation mode, deprecation compilation level (#26355)
Signed-off-by: morrison-turnansky <mturnans@redhat.com> Signed-off-by: Morrison Turnansky <mturnans@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
e66d787bce
commit
96b9aa5aa0
@@ -11,7 +11,7 @@ from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.monitor import set_cudagraph_capturing_enabled
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationLevel,
|
||||
CompilationMode,
|
||||
CUDAGraphMode,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
@@ -42,7 +42,7 @@ def _create_vllm_config(
|
||||
mock_config.parallel_config = ParallelConfig()
|
||||
|
||||
# Mimic the behavior of VllmConfig.__post_init__()
|
||||
if compilation_config.level == CompilationLevel.PIECEWISE:
|
||||
if compilation_config.mode == CompilationMode.VLLM_COMPILE:
|
||||
compilation_config.set_splitting_ops_for_v1()
|
||||
|
||||
return mock_config
|
||||
@@ -50,23 +50,23 @@ def _create_vllm_config(
|
||||
|
||||
class TestCudagraphDispatcher:
|
||||
@pytest.mark.parametrize(
|
||||
"case_id,cudagraph_mode_str,compilation_level",
|
||||
"case_id,cudagraph_mode_str,compilation_mode",
|
||||
[
|
||||
# Test case 0: Full CG for mixed batches, no separate routine
|
||||
(0, "FULL", CompilationLevel.NO_COMPILATION),
|
||||
(0, "FULL", CompilationMode.NONE),
|
||||
# Test case 1: Full CG for uniform batches, piecewise for mixed
|
||||
(1, "FULL_AND_PIECEWISE", CompilationLevel.NO_COMPILATION),
|
||||
(1, "FULL_AND_PIECEWISE", CompilationMode.NONE),
|
||||
# Test case 2: Full CG for uniform batches, no CG for mixed
|
||||
(2, "FULL_DECODE_ONLY", CompilationLevel.NO_COMPILATION),
|
||||
# Test case 3: Piecewise for all
|
||||
(3, "PIECEWISE", CompilationLevel.PIECEWISE),
|
||||
(2, "FULL_DECODE_ONLY", CompilationMode.NONE),
|
||||
# Test case 3: PIECEWISE for all
|
||||
(3, "PIECEWISE", CompilationMode.VLLM_COMPILE),
|
||||
],
|
||||
)
|
||||
def test_dispatcher(self, cudagraph_mode_str, compilation_level):
|
||||
def test_dispatcher(self, cudagraph_mode_str, compilation_mode):
|
||||
# Setup dispatcher
|
||||
comp_config = CompilationConfig(
|
||||
cudagraph_mode=cudagraph_mode_str,
|
||||
level=compilation_level,
|
||||
mode=compilation_mode,
|
||||
cudagraph_capture_sizes=[1, 8],
|
||||
)
|
||||
|
||||
@@ -242,7 +242,7 @@ class TestCudagraphIntegration:
|
||||
def setup_method(self):
|
||||
# only FULL mode for non-uniform batches
|
||||
self.comp_config = CompilationConfig(
|
||||
level=CompilationLevel.PIECEWISE,
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode="FULL",
|
||||
cudagraph_capture_sizes=[10, 20],
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ import pytest
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from tests.v1.attention.utils import full_cg_backend_configs as backend_configs
|
||||
from vllm import LLM
|
||||
from vllm.config import CompilationConfig
|
||||
from vllm.config import CompilationConfig, CompilationMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode, supporte
|
||||
gpu_memory_utilization=0.45,
|
||||
max_model_len=1024,
|
||||
compilation_config=CompilationConfig(
|
||||
level=3, cudagraph_mode=cudagraph_mode
|
||||
mode=CompilationMode.VLLM_COMPILE, cudagraph_mode=cudagraph_mode
|
||||
),
|
||||
)
|
||||
llm.generate(["Hello, my name is"] * 10)
|
||||
@@ -90,32 +90,27 @@ def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode, supporte
|
||||
)
|
||||
|
||||
|
||||
# test cudagraph_mode with different compilation level.
|
||||
# (backend_name, cudagraph_mode, compilation_level, supported)
|
||||
# test cudagraph_mode with different compilation mode.
|
||||
# (backend_name, cudagraph_mode, compilation_mode, supported)
|
||||
combo_cases_2 = [
|
||||
("FA2", "FULL", 0, True), # no compilation + full cudagraph
|
||||
("FA2", "FULL", 3, True), # piecewise compilation + full cudagraph
|
||||
("FA2", "PIECEWISE", 0, False), # no compilation + piecewise cudagraph
|
||||
("FA2", "PIECEWISE", 3, True), # piecewise compilation + piecewise cudagraph
|
||||
(
|
||||
"FA2",
|
||||
"FULL_AND_PIECEWISE",
|
||||
0,
|
||||
False,
|
||||
), # piecewise cudagraph not supported without piecewise compilation
|
||||
("FA2", "FULL_AND_PIECEWISE", 3, True),
|
||||
("FA2", "FULL_DECODE_ONLY", 0, True),
|
||||
("FA2", "FULL_DECODE_ONLY", 3, True),
|
||||
("FA2", "NONE", 0, True), # no compilation + no cudagraph
|
||||
("FA2", "NONE", 3, True), # piecewise compilation + no cudagraph
|
||||
("FA2", "FULL", CompilationMode.NONE, True),
|
||||
("FA2", "FULL", CompilationMode.VLLM_COMPILE, True),
|
||||
("FA2", "PIECEWISE", CompilationMode.NONE, False),
|
||||
("FA2", "PIECEWISE", CompilationMode.VLLM_COMPILE, True),
|
||||
("FA2", "FULL_AND_PIECEWISE", CompilationMode.NONE, False),
|
||||
("FA2", "FULL_AND_PIECEWISE", CompilationMode.VLLM_COMPILE, True),
|
||||
("FA2", "FULL_DECODE_ONLY", CompilationMode.NONE, True),
|
||||
("FA2", "FULL_DECODE_ONLY", CompilationMode.VLLM_COMPILE, True),
|
||||
("FA2", "NONE", CompilationMode.NONE, True),
|
||||
("FA2", "NONE", CompilationMode.VLLM_COMPILE, True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,cudagraph_mode,compilation_level,supported", combo_cases_2
|
||||
"backend_name,cudagraph_mode,compilation_mode,supported", combo_cases_2
|
||||
)
|
||||
def test_cudagraph_compilation_combo(combo_case):
|
||||
backend_name, cudagraph_mode, compilation_level, supported = combo_case
|
||||
backend_name, cudagraph_mode, compilation_mode, supported = combo_case
|
||||
|
||||
env_vars = backend_configs[backend_name].env_vars
|
||||
|
||||
@@ -130,7 +125,7 @@ def test_cudagraph_compilation_combo(combo_case):
|
||||
gpu_memory_utilization=0.45,
|
||||
max_model_len=1024,
|
||||
compilation_config=CompilationConfig(
|
||||
level=compilation_level, cudagraph_mode=cudagraph_mode
|
||||
mode=compilation_mode, cudagraph_mode=cudagraph_mode
|
||||
),
|
||||
)
|
||||
llm.generate(["Hello, my name is"] * 10)
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import CompilationConfig, CompilationLevel
|
||||
from vllm.config import CompilationConfig, CompilationMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from ...utils import fork_new_process_for_each_test
|
||||
@@ -75,9 +75,9 @@ def test_kv_sharing_fast_prefill(
|
||||
# This allows vLLM compilation backend to handle allocating and
|
||||
# managing buffers for cudagraph
|
||||
cudagraph_copy_inputs=True,
|
||||
level=CompilationLevel.PIECEWISE
|
||||
mode=CompilationMode.VLLM_COMPILE
|
||||
if not enforce_eager
|
||||
else CompilationLevel.NO_COMPILATION,
|
||||
else CompilationMode.NONE,
|
||||
)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
|
||||
Reference in New Issue
Block a user