[Feature] Batch invariant torch.compile (#27660)
Signed-off-by: PaulZhang12 <paulzhan@fb.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
+7
-1
@@ -251,6 +251,9 @@ def disable_compile_cache() -> bool:
|
||||
|
||||
|
||||
def use_aot_compile() -> bool:
|
||||
from vllm.model_executor.layers.batch_invariant import (
|
||||
vllm_is_batch_invariant,
|
||||
)
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
default_value = (
|
||||
@@ -259,7 +262,10 @@ def use_aot_compile() -> bool:
|
||||
else "0"
|
||||
)
|
||||
|
||||
return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1"
|
||||
return (
|
||||
not vllm_is_batch_invariant()
|
||||
and os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1"
|
||||
)
|
||||
|
||||
|
||||
def env_with_choices(
|
||||
|
||||
Reference in New Issue
Block a user