[MRV2] Support stock torch.compile mode (#59079)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Wentao Ye
parent
4406874e7c
commit
4e59d53ab5
@@ -12,7 +12,13 @@ from typing import Any, ParamSpec, TypeVar
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CompilationMode, CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import (
|
||||
CompilationMode,
|
||||
CUDAGraphMode,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.compilation import DynamicShapesType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.nvtx_pytorch_hooks import layerwise_nvtx_marker_context
|
||||
@@ -44,6 +50,17 @@ def _compilation_context() -> Generator[None, None, None]:
|
||||
torch._dynamo.config.accumulated_cache_size_limit = original_accumulated_cache
|
||||
|
||||
|
||||
def compile_model_with_stock_torch(
|
||||
model: torch.nn.Module, vllm_config: VllmConfig
|
||||
) -> None:
|
||||
from vllm.env_override import _apply_constrain_to_fx_strides_patch
|
||||
|
||||
_apply_constrain_to_fx_strides_patch()
|
||||
backend = vllm_config.compilation_config.init_backend(vllm_config)
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
|
||||
class TorchCompileWithNoGuardsWrapper:
|
||||
"""A wrapper class for torch.compile, it ensures that all guards are dropped
|
||||
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
|
||||
|
||||
Reference in New Issue
Block a user