[Model] Remove Mono-InternVL (InternLM2VEForCausalLM) (#45129)

Signed-off-by: Xianbao QIAN <xianbao.qian@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
2026-06-12 10:41:09 +00:00
parent 88ed636218
commit f1e13f7df9
9 changed files with 53 additions and 262 deletions
@@ -577,7 +577,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ |
 | `InternS1ProForConditionalGeneration` | Intern-S1-Pro | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1-Pro`, etc. | ✅︎ | ✅︎ |
 | `InternS2PreviewForConditionalGeneration` | Intern-S2-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S2-Preview`, etc. | ✅︎ | ✅︎ |
-| `InternVLChatModel` | InternVL 3.5, InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3_5-14B`, `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ |
+| `InternVLChatModel` | InternVL 3.5, InternVL 3.0, InternVideo 2.5, InternVL 2.5, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3_5-14B`, `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ |
 | `InternVLForConditionalGeneration` | InternVL 3.0 (HF format) | T + I<sup>E+</sup> + V<sup>E+</sup> | `OpenGVLab/InternVL3-1B-hf`, etc. | ✅︎ | ✅︎ |
 | `KananaVForConditionalGeneration` | Kanana-V | T + I<sup>+</sup> | `kakaocorp/kanana-1.5-v-3b-instruct`, etc. | | ✅︎ |
 | `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | ✅︎ | ✅︎ |
@@ -604,8 +604,6 @@ VLM_TEST_SETTINGS = {
        models=[
            "OpenGVLab/InternVL2-1B",
            "OpenGVLab/InternVL2-2B",
-            # FIXME: Config cannot be loaded in transformers 4.52
-            # "OpenGVLab/Mono-InternVL-2B",
        ],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n",  # noqa: E501
@@ -341,17 +341,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
    "InternLM2ForCausalLM": _HfExamplesInfo(
        "internlm/internlm2-chat-7b", trust_remote_code=True
    ),
-    "InternLM2VEForCausalLM": _HfExamplesInfo(
-        "OpenGVLab/Mono-InternVL-2B",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": (
-                "Custom config cannot be loaded with Transformers "
-                "v5 because `vision_config` is not always set"
-            )
-        },
-    ),
    "InternLM3ForCausalLM": _HfExamplesInfo(
        "internlm/internlm3-8b-instruct", trust_remote_code=True
    ),
@@ -157,27 +157,22 @@ class H2OVLChatModel(InternVLChatModel):
        config: PretrainedConfig,
        quant_config: QuantizationConfig | None,
        *,
-        is_mono: bool,
        prefix: str,
    ):
-        if not is_mono:
-            vision_feature_layer = config.select_layer
-            if vision_feature_layer < 0:
-                num_hidden_layers = (
-                    config.vision_config.num_hidden_layers + vision_feature_layer + 1
-                )
-            else:
-                num_hidden_layers = vision_feature_layer + 1
-
-            return InternVisionModel(
-                config.vision_config,
-                quant_config=quant_config,
-                num_hidden_layers_override=num_hidden_layers,
-                prefix=prefix,
+        vision_feature_layer = config.select_layer
+        if vision_feature_layer < 0:
+            num_hidden_layers = (
+                config.vision_config.num_hidden_layers + vision_feature_layer + 1
            )
        else:
-            msg = "Monolith mode is not applicable to H2OVL"
-            raise NotImplementedError(msg)
+            num_hidden_layers = vision_feature_layer + 1
+
+        return InternVisionModel(
+            config.vision_config,
+            quant_config=quant_config,
+            num_hidden_layers_override=num_hidden_layers,
+            prefix=prefix,
+        )

    def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
        if num_image_tokens <= 0 or self.num_image_token <= 0:
@@ -1,139 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from itertools import islice
-
-import torch
-from torch import nn
-from transformers import PretrainedConfig
-
-from vllm.config import CacheConfig, VllmConfig
-from vllm.distributed import get_pp_group
-from vllm.model_executor.layers.layernorm import RMSNorm
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.models.internlm2 import (
-    InternLM2Attention,
-    InternLM2ForCausalLM,
-    InternLM2MLP,
-    InternLM2Model,
-)
-from vllm.sequence import IntermediateTensors
-
-
-class InternLM2VEDecoderLayer(nn.Module):
-    def __init__(
-        self,
-        config: PretrainedConfig,
-        cache_config: CacheConfig | None = None,
-        quant_config: QuantizationConfig | None = None,
-        prefix: str = "",
-    ) -> None:
-        super().__init__()
-        self.hidden_size = config.hidden_size
-        max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
-        self.attention = InternLM2Attention(
-            hidden_size=self.hidden_size,
-            num_heads=config.num_attention_heads,
-            num_kv_heads=config.num_key_value_heads,
-            rope_parameters=config.rope_parameters,
-            max_position_embeddings=max_position_embeddings,
-            cache_config=cache_config,
-            quant_config=quant_config,
-            prefix=f"{prefix}.attention",
-        )
-        self.feed_forward = InternLM2MLP(
-            hidden_size=self.hidden_size,
-            intermediate_size=config.intermediate_size,
-            hidden_act=config.hidden_act,
-            quant_config=quant_config,
-            prefix=f"{prefix}.feed_forward",
-        )
-        self.feed_forward_ve = InternLM2MLP(
-            hidden_size=self.hidden_size,
-            intermediate_size=config.intermediate_size,
-            hidden_act=config.hidden_act,
-            quant_config=quant_config,
-            prefix=f"{prefix}.feed_forward_ve",
-        )
-        self.attention_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-
-    def forward(
-        self,
-        positions: torch.Tensor,
-        hidden_states: torch.Tensor,
-        residual: torch.Tensor | None,
-        visual_token_mask: torch.Tensor | None = None,
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        # Self Attention
-        if residual is None:
-            residual = hidden_states
-            hidden_states = self.attention_norm(hidden_states)
-        else:
-            hidden_states, residual = self.attention_norm(hidden_states, residual)
-        hidden_states = self.attention(
-            positions=positions,
-            hidden_states=hidden_states,
-        )
-
-        # Fully Connected
-        hidden_states, residual = self.ffn_norm(hidden_states, residual)
-        if visual_token_mask is not None and visual_token_mask.any():
-            visual_token_mask = visual_token_mask.repeat(1, self.hidden_size).bool()
-            text_token_mask = ~visual_token_mask
-            hidden_states[visual_token_mask] = self.feed_forward_ve(
-                hidden_states[visual_token_mask].reshape(-1, self.hidden_size)
-            ).flatten()
-            if text_token_mask.any():
-                hidden_states[text_token_mask] = self.feed_forward(
-                    hidden_states[text_token_mask].reshape(-1, self.hidden_size)
-                ).flatten()
-        else:
-            hidden_states = self.feed_forward(hidden_states)
-        return hidden_states, residual
-
-
-class InternLM2VEModel(InternLM2Model):
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        super().__init__(
-            vllm_config=vllm_config, prefix=prefix, layer_type=InternLM2VEDecoderLayer
-        )
-
-    def forward(
-        self,
-        input_ids: torch.Tensor | None,
-        positions: torch.Tensor,
-        intermediate_tensors: IntermediateTensors | None = None,
-        inputs_embeds: torch.Tensor | None = None,
-        visual_token_mask: torch.Tensor | None = None,
-    ) -> torch.Tensor | IntermediateTensors:
-        if get_pp_group().is_first_rank:
-            if inputs_embeds is not None:
-                hidden_states = inputs_embeds
-            else:
-                hidden_states = self.tok_embeddings(input_ids)
-            residual = None
-        else:
-            assert intermediate_tensors is not None
-            hidden_states = intermediate_tensors["hidden_states"]
-            residual = intermediate_tensors["residual"]
-        for layer in islice(self.layers, self.start_layer, self.end_layer):
-            hidden_states, residual = layer(
-                positions,
-                hidden_states,
-                residual,
-                visual_token_mask=visual_token_mask,
-            )
-        if not get_pp_group().is_last_rank:
-            return IntermediateTensors(
-                {"hidden_states": hidden_states, "residual": residual}
-            )
-        hidden_states, _ = self.norm(hidden_states, residual)
-        return hidden_states
-
-
-class InternLM2VEForCausalLM(InternLM2ForCausalLM):
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        super().__init__(
-            vllm_config=vllm_config, prefix=prefix, model_type=InternLM2VEModel
-        )
@@ -23,7 +23,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.quantization.awq import AWQConfig
 from vllm.model_executor.models.intern_vit import (
    InternVisionModel,
-    InternVisionPatchModel,
 )
 from vllm.model_executor.models.module_mapping import MultiModelKeys
 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -582,14 +581,10 @@ class InternVLChatModel(
        self.downsample_ratio = config.downsample_ratio
        self.ps_version = config.ps_version

-        llm_arch_name = config.text_config.architectures[0]
-        self.is_mono = llm_arch_name == "InternLM2VEForCausalLM"
-
        with self._mark_tower_model(vllm_config, {"image", "video"}):
            self.vision_model = self._init_vision_model(
                config,
                quant_config=quant_config,
-                is_mono=self.is_mono,
                prefix=maybe_prefix(prefix, "vision_model"),
            )
            self.mlp1 = self._init_mlp1(config)
@@ -604,7 +599,6 @@ class InternVLChatModel(
        self.img_context_token_id = None
        self.video_context_token_id = None

-        self.visual_token_mask = None
        self.make_empty_intermediate_tensors = (
            self.language_model.make_empty_intermediate_tensors
        )
@@ -627,26 +621,22 @@ class InternVLChatModel(
        config: PretrainedConfig,
        quant_config: QuantizationConfig | None,
        *,
-        is_mono: bool,
        prefix: str,
    ):
-        if not is_mono:
-            vision_feature_layer = config.select_layer
-            if vision_feature_layer < 0:
-                num_hidden_layers = (
-                    config.vision_config.num_hidden_layers + vision_feature_layer + 1
-                )
-            else:
-                num_hidden_layers = vision_feature_layer + 1
-
-            return InternVisionModel(
-                config.vision_config,
-                quant_config=quant_config,
-                num_hidden_layers_override=num_hidden_layers,
-                prefix=prefix,
+        vision_feature_layer = config.select_layer
+        if vision_feature_layer < 0:
+            num_hidden_layers = (
+                config.vision_config.num_hidden_layers + vision_feature_layer + 1
            )
        else:
-            return InternVisionPatchModel(config.vision_config)
+            num_hidden_layers = vision_feature_layer + 1
+
+        return InternVisionModel(
+            config.vision_config,
+            quant_config=quant_config,
+            num_hidden_layers_override=num_hidden_layers,
+            prefix=prefix,
+        )

    def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
        vit_hidden_size = config.vision_config.hidden_size
@@ -805,15 +795,6 @@ class InternVLChatModel(

        return modalities

-    def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
-        if self.is_mono:
-            assert self.img_context_token_id is not None
-            self.visual_token_mask = (input_ids == self.img_context_token_id).reshape(
-                -1, 1
-            )
-        else:
-            self.visual_token_mask = None
-
    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
        modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
        if not modalities:
@@ -844,9 +825,6 @@ class InternVLChatModel(
        *,
        is_multimodal: torch.Tensor | None = None,
    ) -> torch.Tensor:
-        if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
-            self._set_visual_token_mask(input_ids)
-
        # This is to satisfy the type checker for each overload
        if multimodal_embeddings is None or is_multimodal is None:
            return super().embed_input_ids(input_ids)
@@ -875,11 +853,6 @@ class InternVLChatModel(
            "inputs_embeds": inputs_embeds,
        }

-        # Only required if the model is mono-architecture
-        if self.visual_token_mask is not None:
-            forward_kwargs.update({"visual_token_mask": self.visual_token_mask})
-            self.visual_token_mask = None
-
        hidden_states = self.language_model.model(**forward_kwargs)
        return hidden_states

@@ -177,27 +177,22 @@ class NVLM_D_Model(InternVLChatModel):
        config: PretrainedConfig,
        quant_config: QuantizationConfig | None,
        *,
-        is_mono: bool,
        prefix: str,
    ):
-        if not is_mono:
-            vision_feature_layer = config.select_layer
-            if vision_feature_layer < 0:
-                num_hidden_layers = (
-                    config.vision_config.num_hidden_layers + vision_feature_layer + 1
-                )
-            else:
-                num_hidden_layers = vision_feature_layer + 1
-
-            # We added additional dummy heads to the original num of heads to
-            # make the number of heads divisible by 8.
-            return InternVisionModel(
-                config.vision_config,
-                quant_config=quant_config,
-                num_hidden_layers_override=num_hidden_layers,
-                num_dummy_heads=7,
-                prefix=prefix,
+        vision_feature_layer = config.select_layer
+        if vision_feature_layer < 0:
+            num_hidden_layers = (
+                config.vision_config.num_hidden_layers + vision_feature_layer + 1
            )
        else:
-            msg = "Monolith mode is not applicable to NVLM_D"
-            raise NotImplementedError(msg)
+            num_hidden_layers = vision_feature_layer + 1
+
+        # We added additional dummy heads to the original num of heads to
+        # make the number of heads divisible by 8.
+        return InternVisionModel(
+            config.vision_config,
+            quant_config=quant_config,
+            num_hidden_layers_override=num_hidden_layers,
+            num_dummy_heads=7,
+            prefix=prefix,
+        )
@@ -141,7 +141,6 @@ _TEXT_GENERATION_MODELS = {
    "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
    "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
-    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
@@ -716,6 +715,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
    "ErnieModel": "0.23.0",
    "ErnieForSequenceClassification": "0.23.0",
    "ErnieForTokenClassification": "0.23.0",
+    "InternLM2VEForCausalLM": "0.23.0",
    "QWenLMHeadModel": "0.23.0",
    "QwenVLForConditionalGeneration": "0.23.0",
    "InternLMForCausalLM": "0.23.0",
@@ -22,7 +22,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.quantization.awq import AWQConfig
 from vllm.model_executor.models.intern_vit import (
    InternVisionModel,
-    InternVisionPatchModel,
 )
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.processing import BaseDummyInputsBuilder
@@ -178,14 +177,10 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
        self.downsample_ratio = config.downsample_ratio
        self.ps_version = config.ps_version

-        llm_arch_name = config.text_config.architectures[0]
-        self.is_mono = llm_arch_name == "SkyworkLM2VEForCausalLM"
-
        with self._mark_tower_model(vllm_config, "image"):
            self.vision_model = self._init_vision_model(
                config,
                quant_config=quant_config,
-                is_mono=self.is_mono,
                prefix=maybe_prefix(prefix, "vision_model"),
            )
            self.mlp1 = self._init_mlp1(
@@ -223,26 +218,22 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
        config: PretrainedConfig,
        quant_config: QuantizationConfig | None,
        *,
-        is_mono: bool,
        prefix: str,
    ):
-        if not is_mono:
-            vision_feature_layer = config.select_layer
-            if vision_feature_layer < 0:
-                num_hidden_layers = (
-                    config.vision_config.num_hidden_layers + vision_feature_layer + 1
-                )
-            else:
-                num_hidden_layers = vision_feature_layer + 1
-
-            return InternVisionModel(
-                config.vision_config,
-                quant_config=quant_config,
-                num_hidden_layers_override=num_hidden_layers,
-                prefix=prefix,
+        vision_feature_layer = config.select_layer
+        if vision_feature_layer < 0:
+            num_hidden_layers = (
+                config.vision_config.num_hidden_layers + vision_feature_layer + 1
            )
        else:
-            return InternVisionPatchModel(config.vision_config)
+            num_hidden_layers = vision_feature_layer + 1
+
+        return InternVisionModel(
+            config.vision_config,
+            quant_config=quant_config,
+            num_hidden_layers_override=num_hidden_layers,
+            prefix=prefix,
+        )

    def _init_mlp1(
        self,
@@ -363,14 +354,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
        ]
        return image_embeds.split(image_feature_sizes)

-    def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
-        if self.is_mono:
-            self.visual_token_mask = (input_ids == self.img_context_token_id).reshape(
-                -1, 1
-            )
-        else:
-            self.visual_token_mask = None
-
    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
        image_input = self._parse_and_validate_image_input(**kwargs)
        if image_input is None:
@@ -385,9 +368,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
        *,
        is_multimodal: torch.Tensor | None = None,
    ) -> torch.Tensor:
-        if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
-            self._set_visual_token_mask(input_ids)
-
        # This is to satisfy the type checker for each overload
        if multimodal_embeddings is None or is_multimodal is None:
            return super().embed_input_ids(input_ids)