AOT Compilation for torch.compile (Bundled) (#24274)
Signed-off-by: zhxchen17 <zhxchen17@fb.com>
This commit is contained in:
@@ -89,6 +89,8 @@ if TYPE_CHECKING:
|
||||
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
|
||||
VLLM_TORCH_PROFILER_RECORD_SHAPES: bool = False
|
||||
VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY: bool = False
|
||||
VLLM_USE_AOT_COMPILE: bool = False
|
||||
VLLM_FORCE_AOT_LOAD: bool = False
|
||||
VLLM_TORCH_PROFILER_WITH_STACK: bool = True
|
||||
VLLM_TORCH_PROFILER_WITH_FLOPS: bool = False
|
||||
VLLM_USE_TRITON_AWQ: bool = False
|
||||
@@ -235,6 +237,13 @@ def maybe_convert_bool(value: Optional[str]) -> Optional[bool]:
|
||||
return bool(int(value))
|
||||
|
||||
|
||||
def use_aot_compile() -> bool:
|
||||
from vllm.utils import is_torch_equal_or_newer
|
||||
|
||||
default_value = "1" if is_torch_equal_or_newer("2.10.0.dev") else "0"
|
||||
return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1"
|
||||
|
||||
|
||||
def env_with_choices(
|
||||
env_name: str,
|
||||
default: Optional[str],
|
||||
@@ -494,6 +503,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Dump fx graphs to the given directory.
|
||||
# It will override CompilationConfig.debug_dump_path if set.
|
||||
"VLLM_DEBUG_DUMP_PATH": lambda: os.environ.get("VLLM_DEBUG_DUMP_PATH", None),
|
||||
# Feature flag to enable/disable AOT compilation. This will ensure
|
||||
# compilation is done in warmup phase and the compilation will be
|
||||
# reused in subsequent calls.
|
||||
"VLLM_USE_AOT_COMPILE": use_aot_compile,
|
||||
# Force vllm to always load AOT compiled models from disk. Failure
|
||||
# to load will result in a hard error when this is enabled.
|
||||
# Will be ignored when VLLM_USE_AOT_COMPILE is disabled.
|
||||
"VLLM_FORCE_AOT_LOAD": lambda: os.environ.get("VLLM_FORCE_AOT_LOAD", "0") == "1",
|
||||
# local rank of the process in the distributed setting, used to determine
|
||||
# the GPU device id
|
||||
"LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")),
|
||||
|
||||
Reference in New Issue
Block a user