[MRV2] Support stock torch.compile mode (#59079)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Wentao Ye
parent
4406874e7c
commit
4e59d53ab5
@@ -31,8 +31,9 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.compilation.wrapper import compile_model_with_stock_torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.distributed.aux_output_connector.worker import (
|
||||
AuxOutputWorkerConnector,
|
||||
get_aux_output_connector,
|
||||
@@ -522,6 +523,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
get_offloader().post_init()
|
||||
|
||||
if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
|
||||
compile_model_with_stock_torch(self.model, self.vllm_config)
|
||||
|
||||
@property
|
||||
def dp_size(self) -> int:
|
||||
# Not cached: elastic EP rewrites parallel_config in place on reconfigure.
|
||||
|
||||
Reference in New Issue
Block a user