[MRV2] Support stock torch.compile mode (#59079)

Signed-off-by: cheesecake <farzanaman99@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Cheese Cake
2026-09-28 20:17:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Nick Hill Wentao Ye
parent 4406874e7c
commit 4e59d53ab5
7 changed files with 107 additions and 13 deletions
+18 -1
View File
@@ -12,7 +12,13 @@ from typing import Any, ParamSpec, TypeVar
import torch
import vllm.envs as envs
from vllm.config import CompilationMode, CUDAGraphMode, get_current_vllm_config
from vllm.compilation.counter import compilation_counter
from vllm.config import (
CompilationMode,
CUDAGraphMode,
VllmConfig,
get_current_vllm_config,
)
from vllm.config.compilation import DynamicShapesType
from vllm.logger import init_logger
from vllm.utils.nvtx_pytorch_hooks import layerwise_nvtx_marker_context
@@ -44,6 +50,17 @@ def _compilation_context() -> Generator[None, None, None]:
torch._dynamo.config.accumulated_cache_size_limit = original_accumulated_cache
def compile_model_with_stock_torch(
model: torch.nn.Module, vllm_config: VllmConfig
) -> None:
from vllm.env_override import _apply_constrain_to_fx_strides_patch
_apply_constrain_to_fx_strides_patch()
backend = vllm_config.compilation_config.init_backend(vllm_config)
compilation_counter.stock_torch_compile_count += 1
model.compile(fullgraph=True, backend=backend)
class TorchCompileWithNoGuardsWrapper:
"""A wrapper class for torch.compile, it ensures that all guards are dropped
when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE.
+23 -3
View File
@@ -2041,6 +2041,7 @@ class VllmConfig:
self._validate_mm_processor_device()
if self.use_v2_model_runner:
self._disable_cudagraphs_for_v2_stock_torch_compile()
self._validate_v2_model_runner()
else:
self._validate_v1_model_runner()
@@ -3045,9 +3046,6 @@ class VllmConfig:
unsupported: list[str] = []
speculative_config = self.speculative_config
if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
unsupported.append("stock torch.compile")
if (
self.compilation_config.pass_config.enable_sp
and self.parallel_config.tensor_parallel_size > 1
@@ -3301,6 +3299,28 @@ class VllmConfig:
"Enable attribution or use --enforce-eager."
)
def _disable_cudagraphs_for_v2_stock_torch_compile(self) -> None:
"""Run stock torch.compile without CUDA graphs in Model Runner V2.
V1 never wraps a stock-compiled model in CUDAGraphWrapper, so it runs
without CUDA graphs. V2's CUDA graph manager would otherwise capture
FULL graphs around the stock-compiled model, so disable them to match.
"""
compilation_config = self.compilation_config
if (
compilation_config.mode != CompilationMode.STOCK_TORCH_COMPILE
or compilation_config.cudagraph_mode == CUDAGraphMode.NONE
):
return
logger.info_once(
"CUDA graphs are not supported with stock torch.compile in Model "
"Runner V2. Overriding cudagraph_mode %s to NONE.",
compilation_config.cudagraph_mode.name,
)
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
compilation_config.max_cudagraph_capture_size = 0
compilation_config.cudagraph_capture_sizes = []
def _validate_v2_model_runner(self) -> None:
"""Check for features not yet supported by the V2 model runner."""
if not HAS_TRITON:
+5 -1
View File
@@ -31,8 +31,9 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.compilation.wrapper import compile_model_with_stock_torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.distributed.aux_output_connector.worker import (
AuxOutputWorkerConnector,
get_aux_output_connector,
@@ -522,6 +523,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
get_offloader().post_init()
if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
compile_model_with_stock_torch(self.model, self.vllm_config)
@property
def dp_size(self) -> int:
# Not cached: elastic EP rewrites parallel_config in place on reconfigure.