[6/N] torch.compile rollout to users (#10437)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
+20
-23
@@ -2174,8 +2174,14 @@ class CompilationConfig(BaseModel):
|
||||
enabled_custom_ops: Counter[str] = PrivateAttr
|
||||
disabled_custom_ops: Counter[str] = PrivateAttr
|
||||
|
||||
@classmethod
|
||||
def from_cli(cls, cli_value: str) -> "CompilationConfig":
|
||||
"""Parse the CLI value for the compilation config."""
|
||||
if cli_value in ["0", "1", "2", "3"]:
|
||||
return cls(level=int(cli_value))
|
||||
return CompilationConfig.model_validate_json(cli_value)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
self.level = envs.VLLM_TORCH_COMPILE_LEVEL
|
||||
|
||||
count_none = self.custom_ops.count("none")
|
||||
count_all = self.custom_ops.count("all")
|
||||
@@ -2249,26 +2255,6 @@ class CompilationConfig(BaseModel):
|
||||
"inductor_specialize_for_cudagraph_no_more_than is None")
|
||||
self.compile_sizes = self.inductor_compile_sizes
|
||||
|
||||
@staticmethod
|
||||
def select_and_init_config() -> "CompilationConfig":
|
||||
"""The order of selecting config is:
|
||||
1. Use the config specified in environment variable.
|
||||
2. Use the config specified in plugins.
|
||||
3. Use the default config.
|
||||
"""
|
||||
config_path = envs.VLLM_TORCH_COMPILE_CONFIG
|
||||
if config_path is not None:
|
||||
with open(config_path) as json_file:
|
||||
config = CompilationConfig.model_validate_json(
|
||||
json_file.read())
|
||||
else:
|
||||
from vllm.plugins import get_compilation_config
|
||||
predefined_config = get_compilation_config()
|
||||
config = predefined_config if predefined_config is not None else (
|
||||
CompilationConfig())
|
||||
|
||||
return config
|
||||
|
||||
|
||||
@dataclass
|
||||
class VllmConfig:
|
||||
@@ -2354,8 +2340,19 @@ class VllmConfig:
|
||||
self.model_config, self.load_config)
|
||||
|
||||
if self.compilation_config is None:
|
||||
self.compilation_config = CompilationConfig.select_and_init_config(
|
||||
)
|
||||
self.compilation_config = CompilationConfig()
|
||||
if envs.VLLM_USE_V1:
|
||||
# NOTE(woosuk): Currently, we use inductor because the piecewise
|
||||
# CUDA graphs do not work properly with the custom CUDA kernels.
|
||||
# FIXME(woosuk): Disable inductor to reduce the compilation time
|
||||
# and avoid any potential issues with the inductor.
|
||||
self.compilation_config.custom_ops = ["none"]
|
||||
self.compilation_config.use_cudagraph = True
|
||||
self.compilation_config.non_cudagraph_ops = [
|
||||
"vllm.unified_v1_flash_attention"
|
||||
]
|
||||
self.compilation_config.use_inductor = True
|
||||
self.compilation_config.enable_fusion = False
|
||||
|
||||
current_platform.check_and_update_config(self)
|
||||
|
||||
|
||||
@@ -8,12 +8,13 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
|
||||
DeviceConfig, HfOverrides, LoadConfig, LoadFormat,
|
||||
LoRAConfig, ModelConfig, ObservabilityConfig,
|
||||
ParallelConfig, PoolerConfig, PromptAdapterConfig,
|
||||
SchedulerConfig, SpeculativeConfig, TaskOption,
|
||||
TokenizerPoolConfig, VllmConfig)
|
||||
from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
|
||||
DecodingConfig, DeviceConfig, HfOverrides, LoadConfig,
|
||||
LoadFormat, LoRAConfig, ModelConfig,
|
||||
ObservabilityConfig, ParallelConfig, PoolerConfig,
|
||||
PromptAdapterConfig, SchedulerConfig,
|
||||
SpeculativeConfig, TaskOption, TokenizerPoolConfig,
|
||||
VllmConfig)
|
||||
from vllm.executor.executor_base import ExecutorBase
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
|
||||
@@ -189,6 +190,7 @@ class EngineArgs:
|
||||
|
||||
override_neuron_config: Optional[Dict[str, Any]] = None
|
||||
override_pooler_config: Optional[PoolerConfig] = None
|
||||
compilation_config: Optional[CompilationConfig] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.tokenizer:
|
||||
@@ -868,6 +870,20 @@ class EngineArgs:
|
||||
help="Override or set the pooling method in the embedding model. "
|
||||
"e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
|
||||
|
||||
parser.add_argument('--compilation-config',
|
||||
'-O',
|
||||
type=CompilationConfig.from_cli,
|
||||
default=None,
|
||||
help='torch.compile configuration for the model.'
|
||||
'When it is a number (0, 1, 2, 3), it will be '
|
||||
'interpreted as the optimization level.\n'
|
||||
'NOTE: level 0 is the default level without '
|
||||
'any optimization. level 1 and 2 are for internal '
|
||||
'testing only. level 3 is the recommended level '
|
||||
'for production.\n'
|
||||
'To specify the full compilation config, '
|
||||
'use a JSON string.')
|
||||
|
||||
return parser
|
||||
|
||||
@classmethod
|
||||
@@ -1142,6 +1158,7 @@ class EngineArgs:
|
||||
decoding_config=decoding_config,
|
||||
observability_config=observability_config,
|
||||
prompt_adapter_config=prompt_adapter_config,
|
||||
compilation_config=self.compilation_config,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -262,7 +262,8 @@ class LLMEngine:
|
||||
"num_scheduler_steps=%d, chunked_prefill_enabled=%s "
|
||||
"multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
|
||||
"use_async_output_proc=%s, use_cached_outputs=%s, "
|
||||
"mm_processor_kwargs=%s, pooler_config=%r)",
|
||||
"mm_processor_kwargs=%s, pooler_config=%r,"
|
||||
"compilation_config=%r",
|
||||
VLLM_VERSION,
|
||||
model_config.model,
|
||||
speculative_config,
|
||||
@@ -297,6 +298,7 @@ class LLMEngine:
|
||||
use_cached_outputs,
|
||||
model_config.mm_processor_kwargs,
|
||||
model_config.pooler_config,
|
||||
vllm_config.compilation_config,
|
||||
)
|
||||
# TODO(woosuk): Print more configs in debug mode.
|
||||
self.model_config = model_config
|
||||
|
||||
@@ -67,8 +67,6 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_TRITON_AWQ: bool = False
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
|
||||
VLLM_SKIP_P2P_CHECK: bool = False
|
||||
VLLM_TORCH_COMPILE_LEVEL: int = 0
|
||||
VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
|
||||
VLLM_DISABLED_KERNELS: List[str] = []
|
||||
VLLM_USE_V1: bool = False
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
|
||||
@@ -209,12 +207,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
|
||||
lambda: bool(
|
||||
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
|
||||
"VLLM_TORCH_COMPILE_LEVEL":
|
||||
lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
|
||||
|
||||
# Path to the config file for torch compile
|
||||
"VLLM_TORCH_COMPILE_CONFIG":
|
||||
lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
|
||||
|
||||
# local rank of the process in the distributed setting, used to determine
|
||||
# the GPU device id
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -40,7 +39,8 @@ class TpuPlatform(Platform):
|
||||
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
|
||||
from vllm.config import CompilationLevel
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
|
||||
if compilation_config.level == CompilationLevel.NO_COMPILATION:
|
||||
# TPU does not support NO_COMPILATION
|
||||
compilation_config.level = CompilationLevel.DYNAMO_ONCE
|
||||
assert compilation_config.level < CompilationLevel.PIECEWISE,\
|
||||
"TPU does not support Inductor."
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -54,18 +54,6 @@ def load_general_plugins():
|
||||
logger.exception("Failed to load plugin %s", plugin.name)
|
||||
|
||||
|
||||
_compilation_config: Optional["CompilationConfig"] = None
|
||||
|
||||
|
||||
def set_compilation_config(config: Optional["CompilationConfig"]):
|
||||
global _compilation_config
|
||||
_compilation_config = config
|
||||
|
||||
|
||||
def get_compilation_config() -> Optional["CompilationConfig"]:
|
||||
return _compilation_config
|
||||
|
||||
|
||||
_current_vllm_config: Optional["VllmConfig"] = None
|
||||
|
||||
|
||||
|
||||
@@ -8,13 +8,12 @@ import torch.distributed
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.compile_context import set_compile_context
|
||||
from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
|
||||
from vllm.config import CompilationLevel, VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.inputs import INPUT_REGISTRY, InputRegistry
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.multimodal import MultiModalKwargs
|
||||
from vllm.plugins import set_compilation_config
|
||||
from vllm.sampling_params import SamplingParams, SamplingType
|
||||
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
|
||||
is_pin_memory_available)
|
||||
@@ -508,20 +507,6 @@ class GPUModelRunner:
|
||||
return model_runner_output
|
||||
|
||||
def load_model(self) -> None:
|
||||
if self.use_cuda_graph:
|
||||
# NOTE(woosuk): Currently, we use inductor because the piecewise
|
||||
# CUDA graphs do not work properly with the custom CUDA kernels.
|
||||
# FIXME(woosuk): Disable inductor to reduce the compilation time
|
||||
# and avoid any potential issues with the inductor.
|
||||
set_compilation_config(
|
||||
CompilationConfig(
|
||||
custom_ops=["none"],
|
||||
use_cudagraph=True,
|
||||
non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
|
||||
use_inductor=True,
|
||||
enable_fusion=False,
|
||||
))
|
||||
|
||||
logger.info("Starting to load model %s...", self.model_config.model)
|
||||
with DeviceMemoryProfiler() as m: # noqa: SIM117
|
||||
self.model = get_model(vllm_config=self.vllm_config)
|
||||
@@ -562,9 +547,8 @@ class GPUModelRunner:
|
||||
def capture_model(self) -> None:
|
||||
if not self.use_cuda_graph:
|
||||
logger.warning(
|
||||
"Skipping CUDA graph capture. Please set "
|
||||
"VLLM_TORCH_COMPILE_LEVEL=%d to use CUDA graphs.",
|
||||
CompilationLevel.PIECEWISE)
|
||||
"Skipping CUDA graph capture. Please add "
|
||||
"-O 3 to use CUDA graphs.", CompilationLevel.PIECEWISE)
|
||||
return
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user