[2/N][torch.compile] make compilation cfg part of vllm cfg (#10383)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
@@ -19,8 +19,7 @@ from vllm.attention import AttentionMetadata, get_attn_backend
|
||||
from vllm.attention.backends.abstract import AttentionState
|
||||
from vllm.attention.backends.utils import CommonAttentionState
|
||||
from vllm.compilation.compile_context import set_compile_context
|
||||
from vllm.compilation.levels import CompilationLevel
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CompilationLevel, VllmConfig
|
||||
from vllm.core.scheduler import SchedulerOutputs
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.distributed.parallel_state import graph_capture
|
||||
@@ -1142,8 +1141,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
"provided. Defaulting to scaling factors of 1.0. "
|
||||
"This may lead to less accurate results!")
|
||||
|
||||
if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS \
|
||||
and supports_dynamo():
|
||||
if self.vllm_config.compilation_config.level ==\
|
||||
CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
|
||||
from vllm.plugins import get_torch_compile_backend
|
||||
backend = get_torch_compile_backend() or "eager"
|
||||
self.model = torch.compile(
|
||||
|
||||
@@ -140,7 +140,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
model = get_model(vllm_config=self.vllm_config)
|
||||
model = model.eval()
|
||||
xm.wait_device_ops()
|
||||
self.model = ModelWrapper(model)
|
||||
self.model = ModelWrapper(model, self.vllm_config)
|
||||
|
||||
def _dummy_run(
|
||||
self,
|
||||
@@ -669,13 +669,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
|
||||
class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
|
||||
|
||||
def __init__(self, model: nn.Module):
|
||||
def __init__(self, model: nn.Module, vllm_config: VllmConfig):
|
||||
self.model = model
|
||||
compiled_callable = torch.compile(self.forward,
|
||||
backend="openxla",
|
||||
fullgraph=True,
|
||||
dynamic=False)
|
||||
super().__init__(compiled_callable)
|
||||
super().__init__(
|
||||
compiled_callable,
|
||||
compilation_level=vllm_config.compilation_config.level)
|
||||
|
||||
def __call__(self, *args, is_prompt: bool, **kwargs):
|
||||
if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher:
|
||||
|
||||
Reference in New Issue
Block a user