[MRV2] Support stock torch.compile mode (#59079)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Wentao Ye
parent
4406874e7c
commit
4e59d53ab5
+23
-3
@@ -2041,6 +2041,7 @@ class VllmConfig:
|
||||
self._validate_mm_processor_device()
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
self._disable_cudagraphs_for_v2_stock_torch_compile()
|
||||
self._validate_v2_model_runner()
|
||||
else:
|
||||
self._validate_v1_model_runner()
|
||||
@@ -3045,9 +3046,6 @@ class VllmConfig:
|
||||
unsupported: list[str] = []
|
||||
speculative_config = self.speculative_config
|
||||
|
||||
if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
|
||||
unsupported.append("stock torch.compile")
|
||||
|
||||
if (
|
||||
self.compilation_config.pass_config.enable_sp
|
||||
and self.parallel_config.tensor_parallel_size > 1
|
||||
@@ -3301,6 +3299,28 @@ class VllmConfig:
|
||||
"Enable attribution or use --enforce-eager."
|
||||
)
|
||||
|
||||
def _disable_cudagraphs_for_v2_stock_torch_compile(self) -> None:
|
||||
"""Run stock torch.compile without CUDA graphs in Model Runner V2.
|
||||
|
||||
V1 never wraps a stock-compiled model in CUDAGraphWrapper, so it runs
|
||||
without CUDA graphs. V2's CUDA graph manager would otherwise capture
|
||||
FULL graphs around the stock-compiled model, so disable them to match.
|
||||
"""
|
||||
compilation_config = self.compilation_config
|
||||
if (
|
||||
compilation_config.mode != CompilationMode.STOCK_TORCH_COMPILE
|
||||
or compilation_config.cudagraph_mode == CUDAGraphMode.NONE
|
||||
):
|
||||
return
|
||||
logger.info_once(
|
||||
"CUDA graphs are not supported with stock torch.compile in Model "
|
||||
"Runner V2. Overriding cudagraph_mode %s to NONE.",
|
||||
compilation_config.cudagraph_mode.name,
|
||||
)
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
compilation_config.max_cudagraph_capture_size = 0
|
||||
compilation_config.cudagraph_capture_sizes = []
|
||||
|
||||
def _validate_v2_model_runner(self) -> None:
|
||||
"""Check for features not yet supported by the V2 model runner."""
|
||||
if not HAS_TRITON:
|
||||
|
||||
Reference in New Issue
Block a user