[torch.compile] integration with compilation control (#9058)
This commit is contained in:
@@ -5,9 +5,11 @@ import tempfile
|
||||
|
||||
import depyf
|
||||
|
||||
from vllm.compilation.levels import CompilationLevel
|
||||
|
||||
# disable custom dispatcher, let Dynamo takes over
|
||||
# all the control
|
||||
os.environ['VLLM_DYNAMO_USE_CUSTOM_DISPATCHER'] = "0"
|
||||
os.environ['VLLM_TORCH_COMPILE_LEVEL'] = str(CompilationLevel.DYNAMO_AS_IS)
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
with depyf.prepare_debug(temp_dir):
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import os
|
||||
|
||||
from vllm.compilation.levels import CompilationLevel
|
||||
|
||||
from ..utils import compare_two_settings
|
||||
|
||||
# --enforce-eager on TPU causes graph compilation
|
||||
@@ -9,8 +11,9 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
|
||||
|
||||
|
||||
def test_custom_dispatcher():
|
||||
compare_two_settings("google/gemma-2b",
|
||||
arg1=["--enforce-eager"],
|
||||
arg2=["--enforce-eager"],
|
||||
env1={"VLLM_DYNAMO_USE_CUSTOM_DISPATCHER": "0"},
|
||||
env2={})
|
||||
compare_two_settings(
|
||||
"google/gemma-2b",
|
||||
arg1=["--enforce-eager"],
|
||||
arg2=["--enforce-eager"],
|
||||
env1={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_ONCE)},
|
||||
env2={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_AS_IS)})
|
||||
|
||||
Reference in New Issue
Block a user