[MRV2] Support stock torch.compile mode (#59079)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Wentao Ye
parent
4406874e7c
commit
4e59d53ab5
@@ -12,7 +12,13 @@ from typing import Any, ParamSpec, TypeVar
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CompilationMode, CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import (
|
||||
CompilationMode,
|
||||
CUDAGraphMode,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.compilation import DynamicShapesType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.nvtx_pytorch_hooks import layerwise_nvtx_marker_context
|
||||
@@ -44,6 +50,17 @@ def _compilation_context() -> Generator[None, None, None]:
|
||||
torch._dynamo.config.accumulated_cache_size_limit = original_accumulated_cache
|
||||
|
||||
|
||||
def compile_model_with_stock_torch(
|
||||
model: torch.nn.Module, vllm_config: VllmConfig
|
||||
) -> None:
|
||||
from vllm.env_override import _apply_constrain_to_fx_strides_patch
|
||||
|
||||
_apply_constrain_to_fx_strides_patch()
|
||||
backend = vllm_config.compilation_config.init_backend(vllm_config)
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
|
||||
class TorchCompileWithNoGuardsWrapper:
|
||||
"""A wrapper class for torch.compile, it ensures that all guards are dropped
|
||||
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
|
||||
|
||||
+23
-3
@@ -2041,6 +2041,7 @@ class VllmConfig:
|
||||
self._validate_mm_processor_device()
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
self._disable_cudagraphs_for_v2_stock_torch_compile()
|
||||
self._validate_v2_model_runner()
|
||||
else:
|
||||
self._validate_v1_model_runner()
|
||||
@@ -3045,9 +3046,6 @@ class VllmConfig:
|
||||
unsupported: list[str] = []
|
||||
speculative_config = self.speculative_config
|
||||
|
||||
if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
|
||||
unsupported.append("stock torch.compile")
|
||||
|
||||
if (
|
||||
self.compilation_config.pass_config.enable_sp
|
||||
and self.parallel_config.tensor_parallel_size > 1
|
||||
@@ -3301,6 +3299,28 @@ class VllmConfig:
|
||||
"Enable attribution or use --enforce-eager."
|
||||
)
|
||||
|
||||
def _disable_cudagraphs_for_v2_stock_torch_compile(self) -> None:
|
||||
"""Run stock torch.compile without CUDA graphs in Model Runner V2.
|
||||
|
||||
V1 never wraps a stock-compiled model in CUDAGraphWrapper, so it runs
|
||||
without CUDA graphs. V2's CUDA graph manager would otherwise capture
|
||||
FULL graphs around the stock-compiled model, so disable them to match.
|
||||
"""
|
||||
compilation_config = self.compilation_config
|
||||
if (
|
||||
compilation_config.mode != CompilationMode.STOCK_TORCH_COMPILE
|
||||
or compilation_config.cudagraph_mode == CUDAGraphMode.NONE
|
||||
):
|
||||
return
|
||||
logger.info_once(
|
||||
"CUDA graphs are not supported with stock torch.compile in Model "
|
||||
"Runner V2. Overriding cudagraph_mode %s to NONE.",
|
||||
compilation_config.cudagraph_mode.name,
|
||||
)
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
compilation_config.max_cudagraph_capture_size = 0
|
||||
compilation_config.cudagraph_capture_sizes = []
|
||||
|
||||
def _validate_v2_model_runner(self) -> None:
|
||||
"""Check for features not yet supported by the V2 model runner."""
|
||||
if not HAS_TRITON:
|
||||
|
||||
@@ -31,8 +31,9 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.compilation.wrapper import compile_model_with_stock_torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.distributed.aux_output_connector.worker import (
|
||||
AuxOutputWorkerConnector,
|
||||
get_aux_output_connector,
|
||||
@@ -522,6 +523,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
get_offloader().post_init()
|
||||
|
||||
if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
|
||||
compile_model_with_stock_torch(self.model, self.vllm_config)
|
||||
|
||||
@property
|
||||
def dp_size(self) -> int:
|
||||
# Not cached: elastic EP rewrites parallel_config in place on reconfigure.
|
||||
|
||||
Reference in New Issue
Block a user