Don't compile vision encoder for Transformers backend (#30518)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -16,6 +16,7 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend base class."""
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
@@ -29,6 +30,7 @@ from torch import nn
|
||||
from transformers import AutoModel
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_pp_group, get_tp_group
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
@@ -47,6 +49,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
can_enable_torch_compile,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
@@ -117,6 +120,7 @@ class Base(
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.text_config = self.config.get_text_config()
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.device_config = vllm_config.device_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
@@ -155,14 +159,16 @@ class Base(
|
||||
if "gptq" in quant_method_name:
|
||||
self.ignore_unexpected_suffixes.append(".bias")
|
||||
|
||||
# Patch config and init on "meta" to delay allocating GPU tensors
|
||||
self._patch_config()
|
||||
from_config_kwargs = dict(
|
||||
config=self.config,
|
||||
dtype=self.model_config.dtype,
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
)
|
||||
self._decorate_for_torch_compile(**from_config_kwargs)
|
||||
# Init on "meta" to delay allocating GPU tensors
|
||||
with init_on_device_without_buffers("meta"):
|
||||
self.model: PreTrainedModel = AutoModel.from_config(
|
||||
self.config,
|
||||
dtype=self.model_config.dtype,
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
)
|
||||
self.model: PreTrainedModel = AutoModel.from_config(**from_config_kwargs)
|
||||
|
||||
# Create weight name to module qualname mapper
|
||||
self._create_hf_to_vllm_mapper()
|
||||
@@ -218,6 +224,82 @@ class Base(
|
||||
if sub_config.dtype != (dtype := self.config.dtype):
|
||||
sub_config.dtype = dtype
|
||||
|
||||
def _get_decoder_cls(self, **kwargs: dict) -> type[PreTrainedModel]:
|
||||
"""
|
||||
Get the decoder class from the model.
|
||||
|
||||
Args:
|
||||
kwargs: The kwargs to create the model.
|
||||
|
||||
Returns:
|
||||
The decoder class.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
model: PreTrainedModel = AutoModel.from_config(**kwargs)
|
||||
decoder_cls = type(model.get_decoder())
|
||||
logger.debug("Identified decoder class as: %s", decoder_cls)
|
||||
del model
|
||||
return decoder_cls
|
||||
|
||||
def _decorate_cls_for_torch_compile(
|
||||
self,
|
||||
cls: type[PreTrainedModel],
|
||||
dynamic_arg_dims: dict[str, int] | None,
|
||||
enable_if: Callable[["VllmConfig"], bool],
|
||||
is_encoder: bool,
|
||||
):
|
||||
"""
|
||||
Decorate `cls` to indicate to vLLM that it supports torch compile.
|
||||
|
||||
Args:
|
||||
cls: The PreTrainedModel class to decorate.
|
||||
dynamic_arg_dims: A mapping from argument name to the dynamic dimensions
|
||||
of the argument. If None, default dynamic arg dims will be used. See
|
||||
[`support_torch_compile`][vllm.compilation.decorators.support_torch_compile]
|
||||
for more details.
|
||||
enable_if: A function which takes in the vLLM config and returns whether
|
||||
torch compile should be enabled for this class.
|
||||
is_encoder: Whether the class being decorated is an encoder.
|
||||
"""
|
||||
logger.debug(
|
||||
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
|
||||
cls.__name__,
|
||||
"encoder" if is_encoder else "decoder",
|
||||
dynamic_arg_dims,
|
||||
)
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims=dynamic_arg_dims,
|
||||
enable_if=enable_if,
|
||||
is_encoder=is_encoder,
|
||||
)
|
||||
class SupportTorchCompileWrapper(cls): ...
|
||||
|
||||
# Patch the class in its module
|
||||
module = sys.modules[cls.__module__]
|
||||
setattr(module, cls.__name__, SupportTorchCompileWrapper)
|
||||
|
||||
def _decorate_for_torch_compile(self, **kwargs: dict):
|
||||
"""
|
||||
Decorate the model's decoder class to indicate to vLLM that it supports torch
|
||||
compile if `can_enable_torch_compile` is True.
|
||||
|
||||
Args:
|
||||
kwargs: The kwargs to create the model, which are needed to get the decoder
|
||||
class.
|
||||
"""
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._get_decoder_cls(**kwargs),
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
dynamic_arg_dims=dict[str, int](
|
||||
input_ids=1, # shape: [1, seq_len]
|
||||
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
|
||||
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
|
||||
),
|
||||
enable_if=can_enable_torch_compile,
|
||||
is_encoder=False,
|
||||
)
|
||||
|
||||
def _create_hf_to_vllm_mapper(self):
|
||||
"""
|
||||
Create a WeightsMapper to map checkpoint weight names to module qualnames.
|
||||
@@ -553,11 +635,6 @@ class Base(
|
||||
input_ids = None
|
||||
inputs_embeds = intermediate_tensors["hidden_states"]
|
||||
|
||||
if input_ids is not None:
|
||||
input_ids = input_ids[None, ...]
|
||||
if inputs_embeds is not None:
|
||||
inputs_embeds = inputs_embeds[None, ...]
|
||||
|
||||
# If the model scales embeddings inside the input embedding layer we must
|
||||
# ensure they are scaled here since VocabParallelEmbedding will not do it
|
||||
if (
|
||||
@@ -568,22 +645,29 @@ class Base(
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
input_ids = None
|
||||
|
||||
if self.model_config.uses_mrope:
|
||||
position_ids = positions[:, None]
|
||||
else:
|
||||
position_ids = positions[None, ...]
|
||||
# Add batch dimension before entering Transformers model
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
# [seq_len] -> [1, seq_len]
|
||||
input_ids = input_ids[None, ...]
|
||||
if inputs_embeds is not None and inputs_embeds.ndim == 2:
|
||||
# [seq_len, hidden_size] -> [1, seq_len, hidden_size]
|
||||
inputs_embeds = inputs_embeds[None, ...]
|
||||
if positions.ndim == 1:
|
||||
# [seq_len] -> [1, seq_len]
|
||||
positions = positions[None, ...]
|
||||
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=False,
|
||||
position_ids=position_ids,
|
||||
position_ids=positions,
|
||||
attention_instances=self.attention_instances,
|
||||
return_dict=False,
|
||||
**self._output_aux_hidden_states_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
# We must remove the batch dimension from these outputs
|
||||
|
||||
# Remove batch dimension after exiting Transformers model
|
||||
hidden_states = outputs[0][0, ...]
|
||||
if self._output_aux_hidden_states_kwargs:
|
||||
aux_hidden_states = [x[0][0, ...] for x in outputs[1:]]
|
||||
|
||||
Reference in New Issue
Block a user