diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 278f2d368c..87edb27f1f 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -626,8 +626,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Step3VLForConditionalGeneration` | Step3-VL | T + I+ | `stepfun-ai/step3` | | ✅︎ | | `StepVLForConditionalGeneration` | Step3-VL-10B | T + I+ | `stepfun-ai/Step3-VL-10B` | | ✅︎ | | `Step3p7ForConditionalGeneration` | Step-3.7-Flash | T + I+ | `stepfun-ai/Step-3.7-Flash` | | ✅︎ | -| `TarsierForConditionalGeneration` | Tarsier | T + IE+ | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ | -| `Tarsier2ForConditionalGeneration`^ | Tarsier2 | T + IE+ + VE+ | `omni-research/Tarsier2-Recap-7b`, `omni-research/Tarsier2-7b-0115` | | ✅︎ | | `UltravoxModel` | Ultravox | T + AE+ | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ | | `UnlimitedOCRForCausalLM` | Unlimited-OCR | T + I+ | `baidu/Unlimited-OCR`, etc. | ✅︎ | ✅︎ | diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py index 0fb0da1ec9..c3bec9d5fd 100644 --- a/examples/generate/multimodal/vision_language_multi_image_offline.py +++ b/examples/generate/multimodal/vision_language_multi_image_offline.py @@ -1275,55 +1275,6 @@ def load_step_vl(question: str, image_urls: list[str]) -> ModelRequestData: ) -def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData: - model_name = "omni-research/Tarsier-7b" - - engine_args = EngineArgs( - model=model_name, - trust_remote_code=True, - max_model_len=4096, - limit_mm_per_prompt={"image": len(image_urls)}, - ) - - prompt = f"USER: {'' * len(image_urls)}\n{question}\n ASSISTANT:" - image_data = [fetch_image(url) for url in image_urls] - - return ModelRequestData( - engine_args=engine_args, - prompt=prompt, - image_data=image_data, - ) - - -def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData: - model_name = "omni-research/Tarsier2-Recap-7b" - - engine_args = EngineArgs( - model=model_name, - trust_remote_code=True, - max_model_len=32768, - limit_mm_per_prompt={"image": len(image_urls)}, - hf_overrides={ - "architectures": ["Tarsier2ForConditionalGeneration"], - "model_type": "tarsier2", - }, - ) - - prompt = ( - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - f"<|im_start|>user\n<|vision_start|>{'<|image_pad|>' * len(image_urls)}" - f"<|vision_end|>{question}<|im_end|>\n" - "<|im_start|>assistant\n" - ) - image_data = [fetch_image(url) for url in image_urls] - - return ModelRequestData( - engine_args=engine_args, - prompt=prompt, - image_data=image_data, - ) - - # GLM-4.1V def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData: model_name = "zai-org/GLM-4.1V-9B-Thinking" @@ -1507,8 +1458,6 @@ model_example_map = { "smolvlm": load_smolvlm, "step3": load_step3, "stepvl": load_step_vl, - "tarsier": load_tarsier, - "tarsier2": load_tarsier2, "glm4_1v": load_glm4_1v, "glm4_5v": load_glm4_5v, "glm4_5v_fp8": load_glm4_5v_fp8, diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index e837625908..30d34ccc61 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -2347,65 +2347,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData: ) -# omni-research/Tarsier-7b -def run_tarsier(questions: list[str], modality: str) -> ModelRequestData: - assert modality == "image" - model_name = "omni-research/Tarsier-7b" - - engine_args = EngineArgs( - model=model_name, - trust_remote_code=True, - max_model_len=4096, - limit_mm_per_prompt={modality: 1}, - ) - prompts = [(f"USER: \n{question} ASSISTANT:") for question in questions] - - return ModelRequestData( - engine_args=engine_args, - prompts=prompts, - ) - - -def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData: - model_name = "omni-research/Tarsier2-Recap-7b" - - mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1} - engine_args = EngineArgs( - model=model_name, - max_model_len=4096, - hf_overrides={ - "architectures": ["Tarsier2ForConditionalGeneration"], - "model_type": "tarsier2", - }, - limit_mm_per_prompt=mm_limit, - ) - - image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>" - video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>" - - if modality == "image": - placeholder = image_placeholder - elif modality == "video": - placeholder = video_placeholder - elif modality == "image+video": - placeholder = image_placeholder + video_placeholder - - prompts = [ - ( - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - f"<|im_start|>user\n{placeholder}" - f"{question}<|im_end|>\n" - "<|im_start|>assistant\n" - ) - for question in questions - ] - - return ModelRequestData( - engine_args=engine_args, - prompts=prompts, - ) - - model_example_map = { "aria": run_aria, "aya_vision": run_aya_vision, @@ -2479,8 +2420,6 @@ model_example_map = { "smolvlm": run_smolvlm, "step3": run_step3, "stepvl": run_step_vl, - "tarsier": run_tarsier, - "tarsier2": run_tarsier2, } diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index b6945fb0aa..30e12e899e 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -993,31 +993,6 @@ VLM_TEST_SETTINGS = { hf_output_post_proc=model_utils.smolvlm_trunc_hf_output, num_logprobs=10, ), - "tarsier": VLMTestInfo( - models=["omni-research/Tarsier-7b"], - test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), - prompt_formatter=lambda img_prompt: f"USER: {img_prompt} ASSISTANT:", - max_model_len=4096, - max_num_seqs=2, - auto_cls=AutoModelForImageTextToText, - patch_hf_runner=model_utils.tarsier_patch_hf_runner, - ), - "tarsier2": VLMTestInfo( - models=["omni-research/Tarsier2-Recap-7b"], - test_type=( - VLMTestType.IMAGE, - VLMTestType.MULTI_IMAGE, - VLMTestType.VIDEO, - ), - prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", - max_model_len=4096, - max_num_seqs=2, - auto_cls=AutoModelForImageTextToText, - image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)], - marks=[pytest.mark.skip("Model initialization hangs")], - ), ### Tensor parallel / multi-gpu broadcast tests "chameleon-broadcast": VLMTestInfo( models=["facebook/chameleon-7b"], diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index e3f08bf923..de9114b2a3 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -1220,18 +1220,6 @@ def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: return hf_model -def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner: - from vllm.model_executor.models.tarsier import get_vision_encoder_info - - vision_encoder_info = get_vision_encoder_info(hf_model.config) - - hf_processor = hf_model.processor - if hf_processor.patch_size is None: - hf_processor.patch_size = vision_encoder_info.get_patch_size() - - return hf_model - - def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner": """Patch HfRunner for Voxtral's conversation-based processor. diff --git a/tests/models/registry.py b/tests/models/registry.py index 0bc68f0f7b..727a12d045 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1369,22 +1369,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "fixie-ai/ultravox-v0_5-llama-3_2-1b", trust_remote_code=True, ), - "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"), - "Tarsier2ForConditionalGeneration": _HfExamplesInfo( - "omni-research/Tarsier2-Recap-7b", - hf_overrides={ - "architectures": ["Tarsier2ForConditionalGeneration"], - "model_type": "tarsier2", - }, - max_transformers_version="5.3", - transformers_version_reason={ - "vllm": ( - "Qwen2VLConfig was split into Qwen2VLConfig + " - "Qwen2VLTextConfig in transformers v5, breaking " - "attribute access (num_attention_heads, hidden_size, etc.)" - ) - }, - ), "VoxtralForConditionalGeneration": _HfExamplesInfo( "mistralai/Voxtral-Mini-3B-2507", tokenizer_mode="mistral", diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index dd7e3cd10a..8625aec7ec 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -41,7 +41,6 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import ( Qwen2VLVisionConfig, ) from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize -from transformers.models.qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -86,7 +85,6 @@ from vllm.multimodal.processing import ( PromptUpdate, ) from vllm.sequence import IntermediateTensors -from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers @@ -1755,83 +1753,3 @@ class Qwen2VLForConditionalGeneration( vision_config = hf_config.vision_config merge_size = vision_config.spatial_merge_size return num_vision_tokens // merge_size**2 - - -class Tarsier2MultiModalProcessor(Qwen2VLMultiModalProcessor): - pass - - -class Tarsier2ImageProcessor(Qwen2VLImageProcessor): - def __init__( - self, - size: dict[str, int] | None = None, - **kwargs, - ) -> None: - if size is not None and "min_pixels" in size and "max_pixels" in size: - # Remap if Tarsier2-specific format is provided - remapped_size = { - "shortest_edge": size["min_pixels"], - "longest_edge": size["max_pixels"], - } - super().__init__(size=remapped_size, **kwargs) - else: - super().__init__(size=size, **kwargs) - - -class Tarsier2Processor(Qwen2VLProcessor): - def __init__( - self, - image_processor: Tarsier2ImageProcessor, - tokenizer: TokenizerLike, - video_processor: Qwen2VLVideoProcessor, - **kwargs, - ): - super().__init__( - image_processor=image_processor, - tokenizer=tokenizer, - video_processor=video_processor, - chat_template=None, - **kwargs, - ) - - -class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo): - def get_hf_config(self) -> Qwen2VLConfig: - model_path = self.ctx.model_config.model - correct_config = Qwen2VLConfig.from_pretrained(model_path) - - return correct_config - - def get_hf_processor(self, **kwargs: object) -> Tarsier2Processor: - vision_config = self.ctx.get_hf_image_processor_config() - image_processor = Tarsier2ImageProcessor(**vision_config) - video_processor = Qwen2VLVideoProcessor(**vision_config) - return Tarsier2Processor( - image_processor=image_processor, - video_processor=video_processor, - tokenizer=self.get_tokenizer(), - **kwargs, - ) - - def get_image_processor(self) -> Tarsier2ImageProcessor: - return Tarsier2ImageProcessor(**self.ctx.get_hf_image_processor_config()) - - -@MULTIMODAL_REGISTRY.register_processor( - Tarsier2MultiModalProcessor, - info=Tarsier2ProcessingInfo, - dummy_inputs=Qwen2VLDummyInputsBuilder, -) -class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration): - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={ - "vision_tower.": "visual.", - } - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - skip_prefixes = [] - if self.visual is None: - skip_prefixes.extend(["visual."]) - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index ca812c8ee9..283cd219e9 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -571,11 +571,6 @@ _MULTIMODAL_MODELS = { "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"), "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"), "Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"), - "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), - "Tarsier2ForConditionalGeneration": ( - "qwen2_vl", - "Tarsier2ForConditionalGeneration", - ), "UltravoxModel": ("ultravox", "UltravoxModel"), "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"), "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"), @@ -736,6 +731,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = { "AquilaForCausalLM": "0.24.0", "Grok1ModelForCausalLM": "0.24.0", "Grok1ForCausalLM": "0.24.0", + "TarsierForConditionalGeneration": "0.24.0", + "Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4 } _OOT_SUPPORTED_MODELS = { diff --git a/vllm/model_executor/models/tarsier.py b/vllm/model_executor/models/tarsier.py deleted file mode 100644 index 51612cdaca..0000000000 --- a/vllm/model_executor/models/tarsier.py +++ /dev/null @@ -1,591 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import math -from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar - -import torch -import torch.nn as nn -from transformers import ( - BatchFeature, - CLIPVisionConfig, - PretrainedConfig, - SiglipVisionConfig, -) -from transformers import LlavaConfig as HfLlavaConfig -from transformers.image_utils import ImageInput, get_image_size, to_numpy_array -from transformers.models.llava import LlavaProcessor -from transformers.processing_utils import ProcessingKwargs, Unpack -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput - -from vllm.config import VllmConfig -from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.models.llava import LlavaDummyInputsBuilder -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems -from vllm.multimodal.parse import ( - ImageEmbeddingItems, - ImageProcessorItems, - ImageSize, - MultiModalDataItems, -) -from vllm.multimodal.processing import ( - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, -) -from vllm.sequence import IntermediateTensors -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .clip import CLIPVisionModel -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP -from .siglip import SiglipVisionModel -from .utils import ( - AutoWeightsLoader, - get_layer_index, - init_vllm_registered_model, - maybe_prefix, -) -from .vision import ( - VisionEncoderInfo, - get_num_selected_vision_tokens, - get_vision_encoder_info, -) - - -class TarsierImagePixelInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - c: Number of channels (3) - - h: Height - - w: Width - """ - - type: Literal["pixel_values"] = "pixel_values" - pixel_values: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")] - - -class TarsierImageEmbeddingInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - ifs: Image feature size - - hs: Hidden size (must match the hidden size of language model - backbone) - """ - - type: Literal["image_embeds"] = "image_embeds" - data: Annotated[torch.Tensor, TensorShape("bn", "ifs", "hs")] - - -TarsierImageInputs: TypeAlias = TarsierImagePixelInputs | TarsierImageEmbeddingInputs - - -class TarsierHfConfig(Protocol): # Based on the Tarsier's LlavaConfig - vision_config: Final[PretrainedConfig] - text_config: Final[PretrainedConfig] # Added from Tarsier's LlavaConfig - image_token_index: Final[int] - vision_feature_select_strategy: Final[str] - vision_feature_layer: Final[int | list[int]] - projector_hidden_act: Final[str] - image_newline_idx: Final[int] - image_new_idx: Final[int] - multimodal_projector_bias: bool = True - - -class TarsierProcessorKwargs(ProcessingKwargs, total=False): - _defaults = { - "text_kwargs": { - "padding": False, - }, - "images_kwargs": {}, - } - - -class TarsierProcessor(LlavaProcessor): - def __call__( - self, - images: ImageInput = None, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - audio=None, - videos=None, - **kwargs: Unpack[TarsierProcessorKwargs], - ) -> BatchFeature: - if images is None and text is None: - raise ValueError("You have to specify at least one of `images` or `text`.") - - output_kwargs = self._merge_kwargs( - TarsierProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - if images is not None: - image_inputs = self.image_processor( - images, **output_kwargs["images_kwargs"] - ) - else: - image_inputs = {} - - if isinstance(text, str): - text = [text] - elif not isinstance(text, list) and not isinstance(text[0], str): - raise ValueError( - "Invalid input text. Please provide a string, or a list of strings" - ) - - # try to expand inputs in processing if we have the necessary parts - prompt_strings = text - if image_inputs.get("pixel_values") is not None: - # Replace the image token with the expanded image token sequence - pixel_values = image_inputs["pixel_values"] - height, width = get_image_size(to_numpy_array(pixel_values[0])) - num_image_tokens = ( - (height // self.patch_size) * (width // self.patch_size + 1) - + self.num_additional_image_tokens - + 1 - ) - if self.vision_feature_select_strategy == "default": - num_image_tokens -= 1 - - prompt_strings = [] - for sample in text: - sample = sample.replace( - self.image_token, self.image_token * num_image_tokens - ) - prompt_strings.append(sample) - - return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) - text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"]) - return BatchFeature( - data={**text_inputs, **image_inputs}, tensor_type=return_tensors - ) - - -class TarsierMultiModalProjector(nn.Module): - def __init__( - self, - vision_hidden_size: int, - text_hidden_size: int, - projector_hidden_act: str, - multimodal_projector_bias: bool, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - - self.linear_1 = ColumnParallelLinear( - vision_hidden_size, - text_hidden_size, - bias=multimodal_projector_bias, - quant_config=quant_config, - prefix=f"{prefix}.linear_1", - ) - self.act = get_act_fn(projector_hidden_act) - self.linear_2 = RowParallelLinear( - text_hidden_size, - text_hidden_size, - bias=multimodal_projector_bias, - quant_config=quant_config, - prefix=f"{prefix}.linear_2", - ) - - def forward(self, image_features: torch.Tensor) -> torch.Tensor: - hidden_states, _ = self.linear_1(image_features) - hidden_states = self.act(hidden_states) - hidden_states, _ = self.linear_2(hidden_states) - return hidden_states - - -class TarsierProcessingInfo(BaseProcessingInfo): - def get_hf_config(self) -> TarsierHfConfig: - return self.ctx.get_hf_config(HfLlavaConfig) - - def get_vision_encoder_info(self) -> VisionEncoderInfo: - return get_vision_encoder_info(self.get_hf_config()) - - def get_hf_processor(self, **kwargs: object) -> TarsierProcessor: - vision_info = self.get_vision_encoder_info() - - kwargs.setdefault("patch_size", vision_info.get_patch_size()) - - return self.ctx.get_hf_processor(TarsierProcessor, **kwargs) - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"image": None} - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - ) -> int: - hf_config = self.get_hf_config() - vision_encoder_info = self.get_vision_encoder_info() - num_projected_patches = get_num_selected_vision_tokens( - vision_encoder_info.get_num_image_tokens( - image_width=image_width, - image_height=image_height, - ), - hf_config.vision_feature_select_strategy, - ) - if num_projected_patches <= 0: - default_size = self.get_image_size_with_most_features() - num_projected_patches_default = get_num_selected_vision_tokens( - vision_encoder_info.get_num_image_tokens( - image_width=default_size.width, - image_height=default_size.height, - ), - hf_config.vision_feature_select_strategy, - ) - if num_projected_patches_default <= 0: - raise ValueError("Could not determine a valid number of image patches.") - num_projected_patches = num_projected_patches_default - num_height_patches = int(math.sqrt(num_projected_patches)) - total_image_tokens_for_llm = num_projected_patches + num_height_patches + 1 - return total_image_tokens_for_llm - - def get_image_size_with_most_features(self) -> ImageSize: - vision_encoder_info = self.get_vision_encoder_info() - width = height = vision_encoder_info.get_image_size() - return ImageSize(width=width, height=height) - - def get_max_image_tokens(self) -> int: - target_width, target_height = self.get_image_size_with_most_features() - return self.get_num_image_tokens( - image_width=target_width, - image_height=target_height, - ) - - def get_image_newline_idx(self) -> int: - return self.get_hf_config().image_newline_idx - - def get_image_new_idx(self) -> int: - return self.get_hf_config().image_new_idx - - -_I_Tarsier = TypeVar("_I_Tarsier", bound=TarsierProcessingInfo) - - -class TarsierDummyInputsBuilder(LlavaDummyInputsBuilder[_I_Tarsier]): - pass - - -class TarsierMultiModalProcessor(BaseMultiModalProcessor[_I_Tarsier]): - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - return dict( - pixel_values=MultiModalFieldConfig.batched("image"), - image_embeds=MultiModalFieldConfig.batched("image"), - ) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - hf_config = self.info.get_hf_config() - image_token_id = hf_config.image_token_index # The token ID - - def get_replacement(item_idx: int): - images = mm_items.get_items( - "image", (ImageEmbeddingItems, ImageProcessorItems) - ) - - if isinstance(images, ImageEmbeddingItems): - num_projected_patches = images.get_feature_size(item_idx) - # This assumes num_projected_patches is a perfect square - num_height_patches = int(math.sqrt(num_projected_patches)) - num_final_image_tokens = num_projected_patches + num_height_patches + 1 - else: - image_size = images.get_image_size(item_idx) - num_final_image_tokens = self.info.get_num_image_tokens( - image_width=image_size.width, - image_height=image_size.height, - ) - - return [image_token_id] * num_final_image_tokens - - return [ - PromptReplacement( - modality="image", - target=[image_token_id], # Replace each single token - replacement=get_replacement, - ), - ] - - -def init_vision_tower_for_tarsier( - hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol - quant_config: QuantizationConfig | None, - *, - require_post_norm: bool | None = None, - prefix: str = "", -) -> CLIPVisionModel | SiglipVisionModel: - vision_config = hf_config.vision_config - - feature_layers = hf_config.vision_feature_layer - base_num_hidden_layers = vision_config.num_hidden_layers - - if isinstance(feature_layers, int): - num_hidden_layers_to_init = get_layer_index( - feature_layers, base_num_hidden_layers - ) - elif isinstance(feature_layers, (list, tuple)): - num_hidden_layers_to_init = max( - get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers - ) - else: - raise TypeError( - f"vision_layer_feature type: {type(feature_layers)} is not supported" - ) - - if isinstance(vision_config, CLIPVisionConfig): - return CLIPVisionModel( - vision_config, - quant_config=quant_config, - num_hidden_layers_override=num_hidden_layers_to_init, - require_post_norm=require_post_norm, - prefix=prefix, - ) - elif isinstance(vision_config, SiglipVisionConfig): - return SiglipVisionModel( - vision_config, - quant_config=quant_config, - num_hidden_layers_override=num_hidden_layers_to_init, - require_post_norm=require_post_norm, - prefix=prefix, - ) - - msg = f"Unsupported vision config for Tarsier: {type(vision_config)}" - raise NotImplementedError(msg) - - -@MULTIMODAL_REGISTRY.register_processor( - TarsierMultiModalProcessor, - info=TarsierProcessingInfo, - dummy_inputs=TarsierDummyInputsBuilder, -) -class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): - packed_modules_mapping = { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - "gate_up_proj": ["gate_proj", "up_proj"], - } - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return "" - - raise ValueError("Only image modality is supported") - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__() - - config: TarsierHfConfig = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - - self.config = config # Storing the Tarsier-specific HF config - - with self._mark_tower_model(vllm_config, "image"): - self.vision_tower = init_vision_tower_for_tarsier( - config, - quant_config=quant_config, - require_post_norm=False, - prefix=maybe_prefix(prefix, "vision_tower"), - ) - projector_bias = getattr(config, "multimodal_projector_bias", True) - - self.multi_modal_projector = TarsierMultiModalProjector( - vision_hidden_size=config.vision_config.hidden_size, - text_hidden_size=config.text_config.hidden_size, - projector_hidden_act=config.projector_hidden_act, - multimodal_projector_bias=projector_bias, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "multi_modal_projector"), - ) - self.register_buffer( - "image_newline_idx_tensor", - torch.tensor([config.image_newline_idx], dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "image_new_idx_tensor", - torch.tensor([config.image_new_idx], dtype=torch.long), - persistent=False, - ) - - with self._mark_language_model(vllm_config): - self.language_model = init_vllm_registered_model( - vllm_config=vllm_config, - # Use text_config from Tarsier's main config - hf_config=config.text_config, - prefix=maybe_prefix(prefix, "language_model"), - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> TarsierImageInputs | None: - pixel_values = kwargs.pop("pixel_values", None) - image_embeds = kwargs.pop("image_embeds", None) - - if pixel_values is None and image_embeds is None: - return None - - if pixel_values is not None: - return TarsierImagePixelInputs( - type="pixel_values", - pixel_values=pixel_values, - ) - - if image_embeds is not None: - return TarsierImageEmbeddingInputs( - type="image_embeds", - data=image_embeds, - ) - - raise AssertionError("This line should be unreachable.") - - def _image_pixels_to_features( - self, - vision_tower: CLIPVisionModel | SiglipVisionModel, - pixel_values: torch.Tensor | list[torch.Tensor], - ) -> torch.Tensor | tuple[torch.Tensor, ...]: - # From vLLM LLaVA, vision tower output handling - return vision_tower( - pixel_values, - feature_select_strategy=self.config.vision_feature_select_strategy, - ) - - def _add_tarsier_split_tokens( - self, projected_image_features: torch.Tensor - ) -> torch.Tensor: - """ - Implements Tarsier's `add_split_tokens` logic. - """ - num_images, num_projected_patches, embed_dim = projected_image_features.shape - num_height_patches = int(math.sqrt(num_projected_patches)) - num_width_patches = num_projected_patches // num_height_patches - device = projected_image_features.device - embedding_layer = self.language_model.model.embed_tokens - image_newline_emb = embedding_layer( - self.image_newline_idx_tensor.to(device) - ).squeeze(0) - image_new_emb = embedding_layer(self.image_new_idx_tensor.to(device)).squeeze(0) - try: - current_image_features_grid = projected_image_features.view( - num_images, num_height_patches, num_width_patches, embed_dim - ) - except RuntimeError as e: - raise RuntimeError( - "Cannot reshape projected_image_features" - f" with shape {projected_image_features.shape} " - f"to ({num_images}, {num_height_patches}," - f" {num_width_patches}, {embed_dim}). " - "Ensure num_projected_patches is compatible" - " with a grid structure. " - f"num_projected_patches={num_projected_patches}, " - f"derived num_height_patches={num_height_patches}. " - ) from e - - image_newline_expanded = image_newline_emb.expand( - (num_images, num_height_patches, 1, embed_dim) - ) - features_with_newlines = torch.cat( - [current_image_features_grid, image_newline_expanded], - dim=2, # Concatenate along width dim - ) - new_num_patches_after_newline = num_projected_patches + num_height_patches - features_with_newlines_flat = features_with_newlines.view( - num_images, new_num_patches_after_newline, embed_dim - ) - image_new_expanded = image_new_emb.expand((num_images, 1, embed_dim)) - final_image_features = torch.cat( - [features_with_newlines_flat, image_new_expanded], - dim=1, # Concatenate along patch sequence dim - ) - return final_image_features - - def _process_image_pixels( - self, - inputs: TarsierImagePixelInputs, - ) -> torch.Tensor | tuple[torch.Tensor, ...]: - pixel_values = inputs["pixel_values"] - image_features_selected = self._image_pixels_to_features( - self.vision_tower, pixel_values - ) # type: ignore - if isinstance(image_features_selected, torch.Tensor): - projected_features = self.multi_modal_projector(image_features_selected) - final_features = self._add_tarsier_split_tokens(projected_features) - return final_features - else: - raise TypeError( - f"_image_pixels_to_features type:" - f" {type(image_features_selected)} is not supported" - ) - - def _process_image_input( - self, - image_input: TarsierImageInputs, - ) -> torch.Tensor | tuple[torch.Tensor, ...]: - if image_input["type"] == "image_embeds": - projected_features = image_input["data"] - if isinstance(projected_features, torch.Tensor): - return self._add_tarsier_split_tokens(projected_features) - else: - raise ValueError( - "Incorrect type of image_embeds. " - f"Got type: {type(projected_features)}. " - ) - - return self._process_image_pixels(image_input) - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - image_input = self._parse_and_validate_image_input(**kwargs) - if image_input is None: - return [] - return self._process_image_input(image_input) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.language_model.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - return self.language_model.compute_logits(hidden_states) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 654d11df30..67e3db93b8 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -123,7 +123,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( qwen3_5_moe="Qwen3_5MoeConfig", laguna="LagunaConfig", lfm2_moe="Lfm2MoeConfig", - tarsier2="Tarsier2Config", **{"unlimited-ocr": "UnlimitedOCRConfig"}, ) diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 871cb52490..808a8bf077 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -86,7 +86,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe", "Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe", - "Tarsier2Config": "vllm.transformers_utils.configs.tarsier2", # Special case: DeepseekV3Config is from HuggingFace Transformers "DeepseekV3Config": "transformers", } @@ -161,7 +160,6 @@ __all__ = [ "Qwen3_5TextConfig", "Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig", - "Tarsier2Config", ] diff --git a/vllm/transformers_utils/configs/tarsier2.py b/vllm/transformers_utils/configs/tarsier2.py deleted file mode 100644 index 12ebb4b7f6..0000000000 --- a/vllm/transformers_utils/configs/tarsier2.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import Qwen2VLConfig - - -class Tarsier2Config(Qwen2VLConfig): - """ - Tarsier2's config.json is written such that AutoConfig.from_pretrained will create - a deeply nested config consisting of: - - - LlavaConfig - - Qwen2VLConfig - - Qwen2VLTextConfig - - Qwen2VLVisionConfig - - Qwen2VLConfig - - Qwen2VLTextConfig - - Qwen2VLVisionConfig - - When it should really just be a single Qwen2VLConfig. - - This class is a hack to stop AutoConfig from creating the nested config structure. - """ - - model_type = "tarsier2"