[V1] Refactor model executable interface for multimodal models (#10570)
Signed-off-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
@@ -3,7 +3,7 @@ import re
|
||||
from array import array
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache, partial
|
||||
from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
|
||||
from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead, VocabParallelEmbedding)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.multimodal.utils import cached_get_tokenizer
|
||||
from vllm.platforms import _Backend
|
||||
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
|
||||
@@ -756,6 +757,12 @@ class MolmoModel(nn.Module):
|
||||
make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size))
|
||||
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -1098,19 +1105,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
|
||||
return image_features
|
||||
|
||||
def _merge_multimodal_embeddings(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
image_features: torch.Tensor,
|
||||
image_input_idx: torch.Tensor,
|
||||
seq_len: Union[torch.Tensor, List[torch.Tensor]],
|
||||
) -> torch.Tensor:
|
||||
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None:
|
||||
return None
|
||||
image_features = self._process_image_input(image_input)
|
||||
image_input_idx = image_input["image_input_idx"]
|
||||
seq_len = image_input["seq_len"]
|
||||
batch_size, num_image, num_patch = image_features.shape[:3]
|
||||
assert image_input_idx.shape == (batch_size, num_image, num_patch)
|
||||
|
||||
image_features = image_features.to(inputs_embeds.device)
|
||||
seq_len = seq_len.to(inputs_embeds.device)
|
||||
|
||||
# insert the image feature into the embedding.
|
||||
image_features = image_features.view(batch_size, num_image * num_patch,
|
||||
-1)
|
||||
@@ -1130,12 +1134,24 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
|
||||
image_input_idx = image_input_idx.flatten()[:, None]
|
||||
mat = image_input_idx == torch.arange(
|
||||
seq_len.sum().item(), device=inputs_embeds.device)[None, :]
|
||||
seq_len.sum().item(), device=image_features.device)[None, :]
|
||||
mat = mat.to(image_features.dtype)
|
||||
|
||||
inputs_embeds = inputs_embeds + torch.einsum('nd,nm->md',
|
||||
image_features, mat)
|
||||
# Note: In this original implementation from AI2, the final
|
||||
# vision_embeddings will be always be the same length
|
||||
# of input embedddings, which is not very efficient.
|
||||
# TODO(ywang96): see if this can be optimized.
|
||||
vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
|
||||
return vision_embeddings
|
||||
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: Optional[NestedTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.model.get_input_embeddings(input_ids)
|
||||
if multimodal_embeddings is not None:
|
||||
inputs_embeds = inputs_embeds + multimodal_embeddings
|
||||
return inputs_embeds
|
||||
|
||||
def forward(
|
||||
@@ -1145,39 +1161,27 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
kv_caches: List[torch.Tensor],
|
||||
attn_metadata: AttentionMetadata,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs: object,
|
||||
) -> SamplerOutput:
|
||||
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
else:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
if image_input is not None:
|
||||
inputs_embeds = self.model.embed_tokens(input_ids)
|
||||
image_features = self._process_image_input(image_input)
|
||||
# NOTE: In v1, inputs_embeds is always generated at model runner, this
|
||||
# condition is for v0 compatibility.
|
||||
elif inputs_embeds is None:
|
||||
vision_embeddings = self.get_multimodal_embeddings(**kwargs)
|
||||
inputs_embeds = self.get_input_embeddings(input_ids,
|
||||
vision_embeddings)
|
||||
input_ids = None
|
||||
|
||||
inputs_embeds = self._merge_multimodal_embeddings(
|
||||
inputs_embeds,
|
||||
image_features,
|
||||
image_input["image_input_idx"],
|
||||
image_input["seq_len"],
|
||||
)
|
||||
else:
|
||||
inputs_embeds = self.model.embed_tokens(input_ids)
|
||||
|
||||
# always pass the input via `inputs_embeds`
|
||||
# to make sure the computation graph is consistent
|
||||
# for `torch.compile` integration
|
||||
input_ids = None
|
||||
|
||||
hidden_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
hidden_states = self.model(input_ids,
|
||||
positions,
|
||||
kv_caches,
|
||||
attn_metadata,
|
||||
intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user