[torch.compile][BE][Multimodal] Remove requirement to set_model_tag to avoid cache conflict (#37345)
Signed-off-by: Lucas Kabela <lucaskabela@meta.com>
This commit is contained in:
@@ -118,6 +118,7 @@ def support_torch_compile(
|
||||
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
shape_invariants: Callable[..., None] = lambda *args, **kwargs: None,
|
||||
) -> Callable[[type[_T]], type[_T]] | type[_T]:
|
||||
"""
|
||||
@@ -177,6 +178,11 @@ def support_torch_compile(
|
||||
enforce that dynamo does not specialize on 0/1 values in the case of dummy input
|
||||
such as for vision model compilation
|
||||
|
||||
`is_encoder` marks this module as a portion of an multimodal encoder.
|
||||
When True, the compile range upper bound is set to MAX_INT32 instead of
|
||||
max_num_batched_tokens, since encoder input shapes are unpredictable.
|
||||
This is typically used for vision encoder sub-modules in multimodal models.
|
||||
|
||||
`shape_invariants` is a function that gets compiled right before forward.
|
||||
The function should have the torch._check calls that are needed to set
|
||||
the relationships between different input sizes. For example:
|
||||
@@ -226,6 +232,7 @@ def support_torch_compile(
|
||||
inferred_dynamic_arg_dims,
|
||||
mark_unbacked_dims,
|
||||
enable_if,
|
||||
is_encoder,
|
||||
shape_invariants,
|
||||
)
|
||||
|
||||
@@ -316,6 +323,7 @@ def _support_torch_compile(
|
||||
dynamic_arg_dims: dict[str, int | list[int]],
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
shape_invariants: Callable[..., None] = lambda *args, **kwargs: None,
|
||||
) -> type[_T]:
|
||||
"""
|
||||
@@ -345,8 +353,7 @@ def _support_torch_compile(
|
||||
vllm_config = get_current_vllm_config()
|
||||
|
||||
# NOTE: to support multimodal models (such as encoder),
|
||||
# we may not have vllm_config so we may need to patch
|
||||
# it
|
||||
# we may not have vllm_config so we may need to patch it
|
||||
sig = inspect.signature(old_init)
|
||||
if "vllm_config" in sig.parameters:
|
||||
kwargs["vllm_config"] = vllm_config
|
||||
@@ -374,7 +381,11 @@ def _support_torch_compile(
|
||||
self.compiled = False
|
||||
|
||||
# Handled by monkeypatching `TorchCompileWithNoGuardsWrapper` into base class
|
||||
TorchCompileWithNoGuardsWrapper.__init__(self)
|
||||
TorchCompileWithNoGuardsWrapper.__init__(
|
||||
self,
|
||||
compile_prefix=cls.__name__ if is_encoder else "",
|
||||
is_encoder=is_encoder,
|
||||
)
|
||||
|
||||
cls.__init__ = __init__
|
||||
|
||||
|
||||
@@ -75,8 +75,14 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
return ctx.result
|
||||
return callable_fn(*args, **kwargs)
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
compile_prefix: str = "",
|
||||
is_encoder: bool = False,
|
||||
) -> None:
|
||||
self.compiled = False
|
||||
self._compile_prefix = compile_prefix
|
||||
self._is_encoder = is_encoder
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.vllm_config = vllm_config
|
||||
@@ -87,7 +93,9 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
if mode is None:
|
||||
raise RuntimeError("Compilation mode cannot be NO_COMPILATION")
|
||||
|
||||
backend = vllm_config.compilation_config.init_backend(vllm_config)
|
||||
backend = vllm_config.compilation_config.init_backend(
|
||||
vllm_config, prefix=compile_prefix, is_encoder=is_encoder
|
||||
)
|
||||
options = {}
|
||||
|
||||
if isinstance(backend, str) and backend == "inductor":
|
||||
@@ -332,4 +340,8 @@ def reset_compile_wrapper(model: torch.nn.Module) -> None:
|
||||
compilation_config.local_cache_dir = ""
|
||||
|
||||
model.__class__.forward.__code__ = model.original_code_object()
|
||||
TorchCompileWithNoGuardsWrapper.__init__(model)
|
||||
TorchCompileWithNoGuardsWrapper.__init__(
|
||||
model,
|
||||
compile_prefix=model._compile_prefix,
|
||||
is_encoder=model._is_encoder,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user