[Model] Remove Mono-InternVL (InternLM2VEForCausalLM) (#45129)
Signed-off-by: Xianbao QIAN <xianbao.qian@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Claude
Isotr0py
parent
88ed636218
commit
f1e13f7df9
@@ -23,7 +23,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.models.intern_vit import (
|
||||
InternVisionModel,
|
||||
InternVisionPatchModel,
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -582,14 +581,10 @@ class InternVLChatModel(
|
||||
self.downsample_ratio = config.downsample_ratio
|
||||
self.ps_version = config.ps_version
|
||||
|
||||
llm_arch_name = config.text_config.architectures[0]
|
||||
self.is_mono = llm_arch_name == "InternLM2VEForCausalLM"
|
||||
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
self.vision_model = self._init_vision_model(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
is_mono=self.is_mono,
|
||||
prefix=maybe_prefix(prefix, "vision_model"),
|
||||
)
|
||||
self.mlp1 = self._init_mlp1(config)
|
||||
@@ -604,7 +599,6 @@ class InternVLChatModel(
|
||||
self.img_context_token_id = None
|
||||
self.video_context_token_id = None
|
||||
|
||||
self.visual_token_mask = None
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
@@ -627,26 +621,22 @@ class InternVLChatModel(
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
is_mono: bool,
|
||||
prefix: str,
|
||||
):
|
||||
if not is_mono:
|
||||
vision_feature_layer = config.select_layer
|
||||
if vision_feature_layer < 0:
|
||||
num_hidden_layers = (
|
||||
config.vision_config.num_hidden_layers + vision_feature_layer + 1
|
||||
)
|
||||
else:
|
||||
num_hidden_layers = vision_feature_layer + 1
|
||||
|
||||
return InternVisionModel(
|
||||
config.vision_config,
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers,
|
||||
prefix=prefix,
|
||||
vision_feature_layer = config.select_layer
|
||||
if vision_feature_layer < 0:
|
||||
num_hidden_layers = (
|
||||
config.vision_config.num_hidden_layers + vision_feature_layer + 1
|
||||
)
|
||||
else:
|
||||
return InternVisionPatchModel(config.vision_config)
|
||||
num_hidden_layers = vision_feature_layer + 1
|
||||
|
||||
return InternVisionModel(
|
||||
config.vision_config,
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
|
||||
vit_hidden_size = config.vision_config.hidden_size
|
||||
@@ -805,15 +795,6 @@ class InternVLChatModel(
|
||||
|
||||
return modalities
|
||||
|
||||
def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
|
||||
if self.is_mono:
|
||||
assert self.img_context_token_id is not None
|
||||
self.visual_token_mask = (input_ids == self.img_context_token_id).reshape(
|
||||
-1, 1
|
||||
)
|
||||
else:
|
||||
self.visual_token_mask = None
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
if not modalities:
|
||||
@@ -844,9 +825,6 @@ class InternVLChatModel(
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
|
||||
self._set_visual_token_mask(input_ids)
|
||||
|
||||
# This is to satisfy the type checker for each overload
|
||||
if multimodal_embeddings is None or is_multimodal is None:
|
||||
return super().embed_input_ids(input_ids)
|
||||
@@ -875,11 +853,6 @@ class InternVLChatModel(
|
||||
"inputs_embeds": inputs_embeds,
|
||||
}
|
||||
|
||||
# Only required if the model is mono-architecture
|
||||
if self.visual_token_mask is not None:
|
||||
forward_kwargs.update({"visual_token_mask": self.visual_token_mask})
|
||||
self.visual_token_mask = None
|
||||
|
||||
hidden_states = self.language_model.model(**forward_kwargs)
|
||||
return hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user