iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Model] Remove Tarsier, Tarsier2 (#47143)

Browse Source 
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-30 13:20:33 +00:00
committed by GitHub
parent aab7af0bcb
commit 5dc36a4fa5
12 changed files with 2 additions and 872 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/models/supported_models.md
-2
View File
@@ -626,8 +626,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Step3VLForConditionalGeneration` | Step3-VL | T + I<sup>+</sup> | `stepfun-ai/step3` | | ✅︎ |
| `StepVLForConditionalGeneration` | Step3-VL-10B | T + I<sup>+</sup> | `stepfun-ai/Step3-VL-10B` | | ✅︎ |
| `Step3p7ForConditionalGeneration` | Step-3.7-Flash | T + I<sup>+</sup> | `stepfun-ai/Step-3.7-Flash` | | ✅︎ |
| `TarsierForConditionalGeneration` | Tarsier | T + I<sup>E+</sup> | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ |
| `Tarsier2ForConditionalGeneration`<sup>^</sup> | Tarsier2 | T + I<sup>E+</sup> + V<sup>E+</sup> | `omni-research/Tarsier2-Recap-7b`, `omni-research/Tarsier2-7b-0115` | | ✅︎ |
| `UltravoxModel` | Ultravox | T + A<sup>E+</sup> | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ |
| `UnlimitedOCRForCausalLM` | Unlimited-OCR | T + I<sup>+</sup> | `baidu/Unlimited-OCR`, etc. | ✅︎ | ✅︎ |
examples/generate/multimodal/vision_language_multi_image_offline.py
-51
View File
@@ -1275,55 +1275,6 @@ def load_step_vl(question: str, image_urls: list[str]) -> ModelRequestData:
)
def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData:
model_name = "omni-research/Tarsier-7b"
engine_args = EngineArgs(
model=model_name,
trust_remote_code=True,
max_model_len=4096,
limit_mm_per_prompt={"image": len(image_urls)},
)
prompt = f"USER: {'<image>' * len(image_urls)}\n{question}\n ASSISTANT:"
image_data = [fetch_image(url) for url in image_urls]
return ModelRequestData(
engine_args=engine_args,
prompt=prompt,
image_data=image_data,
)
def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
model_name = "omni-research/Tarsier2-Recap-7b"
engine_args = EngineArgs(
model=model_name,
trust_remote_code=True,
max_model_len=32768,
limit_mm_per_prompt={"image": len(image_urls)},
hf_overrides={
"architectures": ["Tarsier2ForConditionalGeneration"],
"model_type": "tarsier2",
},
)
prompt = (
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
f"<|im_start|>user\n<|vision_start|>{'<|image_pad|>' * len(image_urls)}"
f"<|vision_end|>{question}<|im_end|>\n"
"<|im_start|>assistant\n"
)
image_data = [fetch_image(url) for url in image_urls]
return ModelRequestData(
engine_args=engine_args,
prompt=prompt,
image_data=image_data,
)
# GLM-4.1V
def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData:
model_name = "zai-org/GLM-4.1V-9B-Thinking"
@@ -1507,8 +1458,6 @@ model_example_map = {
"smolvlm": load_smolvlm,
"step3": load_step3,
"stepvl": load_step_vl,
"tarsier": load_tarsier,
"tarsier2": load_tarsier2,
"glm4_1v": load_glm4_1v,
"glm4_5v": load_glm4_5v,
"glm4_5v_fp8": load_glm4_5v_fp8,
examples/generate/multimodal/vision_language_offline.py
-61
View File
@@ -2347,65 +2347,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
)
# omni-research/Tarsier-7b
def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
assert modality == "image"
model_name = "omni-research/Tarsier-7b"
engine_args = EngineArgs(
model=model_name,
trust_remote_code=True,
max_model_len=4096,
limit_mm_per_prompt={modality: 1},
)
prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
return ModelRequestData(
engine_args=engine_args,
prompts=prompts,
)
def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
model_name = "omni-research/Tarsier2-Recap-7b"
mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
engine_args = EngineArgs(
model=model_name,
max_model_len=4096,
hf_overrides={
"architectures": ["Tarsier2ForConditionalGeneration"],
"model_type": "tarsier2",
},
limit_mm_per_prompt=mm_limit,
)
image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
if modality == "image":
placeholder = image_placeholder
elif modality == "video":
placeholder = video_placeholder
elif modality == "image+video":
placeholder = image_placeholder + video_placeholder
prompts = [
(
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
f"<|im_start|>user\n{placeholder}"
f"{question}<|im_end|>\n"
"<|im_start|>assistant\n"
)
for question in questions
]
return ModelRequestData(
engine_args=engine_args,
prompts=prompts,
)
model_example_map = {
"aria": run_aria,
"aya_vision": run_aya_vision,
@@ -2479,8 +2420,6 @@ model_example_map = {
"smolvlm": run_smolvlm,
"step3": run_step3,
"stepvl": run_step_vl,
"tarsier": run_tarsier,
"tarsier2": run_tarsier2,
}
tests/models/multimodal/generation/test_common.py
-25
View File
@@ -993,31 +993,6 @@ VLM_TEST_SETTINGS = {
hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,
num_logprobs=10,
),
"tarsier": VLMTestInfo(
models=["omni-research/Tarsier-7b"],
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
prompt_formatter=lambda img_prompt: f"USER: {img_prompt} ASSISTANT:",
max_model_len=4096,
max_num_seqs=2,
auto_cls=AutoModelForImageTextToText,
patch_hf_runner=model_utils.tarsier_patch_hf_runner,
),
"tarsier2": VLMTestInfo(
models=["omni-research/Tarsier2-Recap-7b"],
test_type=(
VLMTestType.IMAGE,
VLMTestType.MULTI_IMAGE,
VLMTestType.VIDEO,
),
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
max_model_len=4096,
max_num_seqs=2,
auto_cls=AutoModelForImageTextToText,
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
marks=[pytest.mark.skip("Model initialization hangs")],
),
### Tensor parallel / multi-gpu broadcast tests
"chameleon-broadcast": VLMTestInfo(
models=["facebook/chameleon-7b"],
tests/models/multimodal/generation/vlm_utils/model_utils.py
-12
View File
@@ -1220,18 +1220,6 @@ def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
return hf_model
def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
from vllm.model_executor.models.tarsier import get_vision_encoder_info
vision_encoder_info = get_vision_encoder_info(hf_model.config)
hf_processor = hf_model.processor
if hf_processor.patch_size is None:
hf_processor.patch_size = vision_encoder_info.get_patch_size()
return hf_model
def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
"""Patch HfRunner for Voxtral's conversation-based processor.
tests/models/registry.py
-16
View File
@@ -1369,22 +1369,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
trust_remote_code=True,
),
"TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
"Tarsier2ForConditionalGeneration": _HfExamplesInfo(
"omni-research/Tarsier2-Recap-7b",
hf_overrides={
"architectures": ["Tarsier2ForConditionalGeneration"],
"model_type": "tarsier2",
},
max_transformers_version="5.3",
transformers_version_reason={
"vllm": (
"Qwen2VLConfig was split into Qwen2VLConfig + "
"Qwen2VLTextConfig in transformers v5, breaking "
"attribute access (num_attention_heads, hidden_size, etc.)"
)
},
),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
tokenizer_mode="mistral",
vllm/model_executor/models/qwen2_vl.py
-82
View File
@@ -41,7 +41,6 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (
Qwen2VLVisionConfig,
)
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from transformers.models.qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
@@ -86,7 +85,6 @@ from vllm.multimodal.processing import (
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import TokenizerLike
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -1755,83 +1753,3 @@ class Qwen2VLForConditionalGeneration(
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
class Tarsier2MultiModalProcessor(Qwen2VLMultiModalProcessor):
pass
class Tarsier2ImageProcessor(Qwen2VLImageProcessor):
def __init__(
self,
size: dict[str, int] | None = None,
**kwargs,
) -> None:
if size is not None and "min_pixels" in size and "max_pixels" in size:
# Remap if Tarsier2-specific format is provided
remapped_size = {
"shortest_edge": size["min_pixels"],
"longest_edge": size["max_pixels"],
}
super().__init__(size=remapped_size, **kwargs)
else:
super().__init__(size=size, **kwargs)
class Tarsier2Processor(Qwen2VLProcessor):
def __init__(
self,
image_processor: Tarsier2ImageProcessor,
tokenizer: TokenizerLike,
video_processor: Qwen2VLVideoProcessor,
**kwargs,
):
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
video_processor=video_processor,
chat_template=None,
**kwargs,
)
class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self) -> Qwen2VLConfig:
model_path = self.ctx.model_config.model
correct_config = Qwen2VLConfig.from_pretrained(model_path)
return correct_config
def get_hf_processor(self, **kwargs: object) -> Tarsier2Processor:
vision_config = self.ctx.get_hf_image_processor_config()
image_processor = Tarsier2ImageProcessor(**vision_config)
video_processor = Qwen2VLVideoProcessor(**vision_config)
return Tarsier2Processor(
image_processor=image_processor,
video_processor=video_processor,
tokenizer=self.get_tokenizer(),
**kwargs,
)
def get_image_processor(self) -> Tarsier2ImageProcessor:
return Tarsier2ImageProcessor(**self.ctx.get_hf_image_processor_config())
@MULTIMODAL_REGISTRY.register_processor(
Tarsier2MultiModalProcessor,
info=Tarsier2ProcessingInfo,
dummy_inputs=Qwen2VLDummyInputsBuilder,
)
class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"vision_tower.": "visual.",
}
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = []
if self.visual is None:
skip_prefixes.extend(["visual."])
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/registry.py
+2 -5
View File
@@ -571,11 +571,6 @@ _MULTIMODAL_MODELS = {
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
"Tarsier2ForConditionalGeneration": (
"qwen2_vl",
"Tarsier2ForConditionalGeneration",
),
"UltravoxModel": ("ultravox", "UltravoxModel"),
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
"VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
@@ -736,6 +731,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"AquilaForCausalLM": "0.24.0",
"Grok1ModelForCausalLM": "0.24.0",
"Grok1ForCausalLM": "0.24.0",
"TarsierForConditionalGeneration": "0.24.0",
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
}
_OOT_SUPPORTED_MODELS = {
vllm/model_executor/models/tarsier.py
-591
View File
@@ -1,591 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
import torch
import torch.nn as nn
from transformers import (
BatchFeature,
CLIPVisionConfig,
PretrainedConfig,
SiglipVisionConfig,
)
from transformers import LlavaConfig as HfLlavaConfig
from transformers.image_utils import ImageInput, get_image_size, to_numpy_array
from transformers.models.llava import LlavaProcessor
from transformers.processing_utils import ProcessingKwargs, Unpack
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from vllm.config import VllmConfig
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import (
ImageEmbeddingItems,
ImageProcessorItems,
ImageSize,
MultiModalDataItems,
)
from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .clip import CLIPVisionModel
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
get_layer_index,
init_vllm_registered_model,
maybe_prefix,
)
from .vision import (
VisionEncoderInfo,
get_num_selected_vision_tokens,
get_vision_encoder_info,
)
class TarsierImagePixelInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- c: Number of channels (3)
- h: Height
- w: Width
"""
type: Literal["pixel_values"] = "pixel_values"
pixel_values: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
class TarsierImageEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size * number of images
- ifs: Image feature size
- hs: Hidden size (must match the hidden size of language model
backbone)
"""
type: Literal["image_embeds"] = "image_embeds"
data: Annotated[torch.Tensor, TensorShape("bn", "ifs", "hs")]
TarsierImageInputs: TypeAlias = TarsierImagePixelInputs | TarsierImageEmbeddingInputs
class TarsierHfConfig(Protocol): # Based on the Tarsier's LlavaConfig
vision_config: Final[PretrainedConfig]
text_config: Final[PretrainedConfig] # Added from Tarsier's LlavaConfig
image_token_index: Final[int]
vision_feature_select_strategy: Final[str]
vision_feature_layer: Final[int | list[int]]
projector_hidden_act: Final[str]
image_newline_idx: Final[int]
image_new_idx: Final[int]
multimodal_projector_bias: bool = True
class TarsierProcessorKwargs(ProcessingKwargs, total=False):
_defaults = {
"text_kwargs": {
"padding": False,
},
"images_kwargs": {},
}
class TarsierProcessor(LlavaProcessor):
def __call__(
self,
images: ImageInput = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
audio=None,
videos=None,
**kwargs: Unpack[TarsierProcessorKwargs],
) -> BatchFeature:
if images is None and text is None:
raise ValueError("You have to specify at least one of `images` or `text`.")
output_kwargs = self._merge_kwargs(
TarsierProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
image_inputs = self.image_processor(
images, **output_kwargs["images_kwargs"]
)
else:
image_inputs = {}
if isinstance(text, str):
text = [text]
elif not isinstance(text, list) and not isinstance(text[0], str):
raise ValueError(
"Invalid input text. Please provide a string, or a list of strings"
)
# try to expand inputs in processing if we have the necessary parts
prompt_strings = text
if image_inputs.get("pixel_values") is not None:
# Replace the image token with the expanded image token sequence
pixel_values = image_inputs["pixel_values"]
height, width = get_image_size(to_numpy_array(pixel_values[0]))
num_image_tokens = (
(height // self.patch_size) * (width // self.patch_size + 1)
+ self.num_additional_image_tokens
+ 1
)
if self.vision_feature_select_strategy == "default":
num_image_tokens -= 1
prompt_strings = []
for sample in text:
sample = sample.replace(
self.image_token, self.image_token * num_image_tokens
)
prompt_strings.append(sample)
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"])
return BatchFeature(
data={**text_inputs, **image_inputs}, tensor_type=return_tensors
)
class TarsierMultiModalProjector(nn.Module):
def __init__(
self,
vision_hidden_size: int,
text_hidden_size: int,
projector_hidden_act: str,
multimodal_projector_bias: bool,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.linear_1 = ColumnParallelLinear(
vision_hidden_size,
text_hidden_size,
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
)
self.act = get_act_fn(projector_hidden_act)
self.linear_2 = RowParallelLinear(
text_hidden_size,
text_hidden_size,
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
)
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
hidden_states, _ = self.linear_1(image_features)
hidden_states = self.act(hidden_states)
hidden_states, _ = self.linear_2(hidden_states)
return hidden_states
class TarsierProcessingInfo(BaseProcessingInfo):
def get_hf_config(self) -> TarsierHfConfig:
return self.ctx.get_hf_config(HfLlavaConfig)
def get_vision_encoder_info(self) -> VisionEncoderInfo:
return get_vision_encoder_info(self.get_hf_config())
def get_hf_processor(self, **kwargs: object) -> TarsierProcessor:
vision_info = self.get_vision_encoder_info()
kwargs.setdefault("patch_size", vision_info.get_patch_size())
return self.ctx.get_hf_processor(TarsierProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
) -> int:
hf_config = self.get_hf_config()
vision_encoder_info = self.get_vision_encoder_info()
num_projected_patches = get_num_selected_vision_tokens(
vision_encoder_info.get_num_image_tokens(
image_width=image_width,
image_height=image_height,
),
hf_config.vision_feature_select_strategy,
)
if num_projected_patches <= 0:
default_size = self.get_image_size_with_most_features()
num_projected_patches_default = get_num_selected_vision_tokens(
vision_encoder_info.get_num_image_tokens(
image_width=default_size.width,
image_height=default_size.height,
),
hf_config.vision_feature_select_strategy,
)
if num_projected_patches_default <= 0:
raise ValueError("Could not determine a valid number of image patches.")
num_projected_patches = num_projected_patches_default
num_height_patches = int(math.sqrt(num_projected_patches))
total_image_tokens_for_llm = num_projected_patches + num_height_patches + 1
return total_image_tokens_for_llm
def get_image_size_with_most_features(self) -> ImageSize:
vision_encoder_info = self.get_vision_encoder_info()
width = height = vision_encoder_info.get_image_size()
return ImageSize(width=width, height=height)
def get_max_image_tokens(self) -> int:
target_width, target_height = self.get_image_size_with_most_features()
return self.get_num_image_tokens(
image_width=target_width,
image_height=target_height,
)
def get_image_newline_idx(self) -> int:
return self.get_hf_config().image_newline_idx
def get_image_new_idx(self) -> int:
return self.get_hf_config().image_new_idx
_I_Tarsier = TypeVar("_I_Tarsier", bound=TarsierProcessingInfo)
class TarsierDummyInputsBuilder(LlavaDummyInputsBuilder[_I_Tarsier]):
pass
class TarsierMultiModalProcessor(BaseMultiModalProcessor[_I_Tarsier]):
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
image_embeds=MultiModalFieldConfig.batched("image"),
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_config = self.info.get_hf_config()
image_token_id = hf_config.image_token_index # The <IMAGE> token ID
def get_replacement(item_idx: int):
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
)
if isinstance(images, ImageEmbeddingItems):
num_projected_patches = images.get_feature_size(item_idx)
# This assumes num_projected_patches is a perfect square
num_height_patches = int(math.sqrt(num_projected_patches))
num_final_image_tokens = num_projected_patches + num_height_patches + 1
else:
image_size = images.get_image_size(item_idx)
num_final_image_tokens = self.info.get_num_image_tokens(
image_width=image_size.width,
image_height=image_size.height,
)
return [image_token_id] * num_final_image_tokens
return [
PromptReplacement(
modality="image",
target=[image_token_id], # Replace each single <IMAGE> token
replacement=get_replacement,
),
]
def init_vision_tower_for_tarsier(
hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol
quant_config: QuantizationConfig | None,
*,
require_post_norm: bool | None = None,
prefix: str = "",
) -> CLIPVisionModel | SiglipVisionModel:
vision_config = hf_config.vision_config
feature_layers = hf_config.vision_feature_layer
base_num_hidden_layers = vision_config.num_hidden_layers
if isinstance(feature_layers, int):
num_hidden_layers_to_init = get_layer_index(
feature_layers, base_num_hidden_layers
)
elif isinstance(feature_layers, (list, tuple)):
num_hidden_layers_to_init = max(
get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers
)
else:
raise TypeError(
f"vision_layer_feature type: {type(feature_layers)} is not supported"
)
if isinstance(vision_config, CLIPVisionConfig):
return CLIPVisionModel(
vision_config,
quant_config=quant_config,
num_hidden_layers_override=num_hidden_layers_to_init,
require_post_norm=require_post_norm,
prefix=prefix,
)
elif isinstance(vision_config, SiglipVisionConfig):
return SiglipVisionModel(
vision_config,
quant_config=quant_config,
num_hidden_layers_override=num_hidden_layers_to_init,
require_post_norm=require_post_norm,
prefix=prefix,
)
msg = f"Unsupported vision config for Tarsier: {type(vision_config)}"
raise NotImplementedError(msg)
@MULTIMODAL_REGISTRY.register_processor(
TarsierMultiModalProcessor,
info=TarsierProcessingInfo,
dummy_inputs=TarsierDummyInputsBuilder,
)
class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
raise ValueError("Only image modality is supported")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config: TarsierHfConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config # Storing the Tarsier-specific HF config
with self._mark_tower_model(vllm_config, "image"):
self.vision_tower = init_vision_tower_for_tarsier(
config,
quant_config=quant_config,
require_post_norm=False,
prefix=maybe_prefix(prefix, "vision_tower"),
)
projector_bias = getattr(config, "multimodal_projector_bias", True)
self.multi_modal_projector = TarsierMultiModalProjector(
vision_hidden_size=config.vision_config.hidden_size,
text_hidden_size=config.text_config.hidden_size,
projector_hidden_act=config.projector_hidden_act,
multimodal_projector_bias=projector_bias,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "multi_modal_projector"),
)
self.register_buffer(
"image_newline_idx_tensor",
torch.tensor([config.image_newline_idx], dtype=torch.long),
persistent=False,
)
self.register_buffer(
"image_new_idx_tensor",
torch.tensor([config.image_new_idx], dtype=torch.long),
persistent=False,
)
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
# Use text_config from Tarsier's main config
hf_config=config.text_config,
prefix=maybe_prefix(prefix, "language_model"),
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> TarsierImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
if pixel_values is None and image_embeds is None:
return None
if pixel_values is not None:
return TarsierImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
)
if image_embeds is not None:
return TarsierImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
raise AssertionError("This line should be unreachable.")
def _image_pixels_to_features(
self,
vision_tower: CLIPVisionModel | SiglipVisionModel,
pixel_values: torch.Tensor | list[torch.Tensor],
) -> torch.Tensor | tuple[torch.Tensor, ...]:
# From vLLM LLaVA, vision tower output handling
return vision_tower(
pixel_values,
feature_select_strategy=self.config.vision_feature_select_strategy,
)
def _add_tarsier_split_tokens(
self, projected_image_features: torch.Tensor
) -> torch.Tensor:
"""
Implements Tarsier's `add_split_tokens` logic.
"""
num_images, num_projected_patches, embed_dim = projected_image_features.shape
num_height_patches = int(math.sqrt(num_projected_patches))
num_width_patches = num_projected_patches // num_height_patches
device = projected_image_features.device
embedding_layer = self.language_model.model.embed_tokens
image_newline_emb = embedding_layer(
self.image_newline_idx_tensor.to(device)
).squeeze(0)
image_new_emb = embedding_layer(self.image_new_idx_tensor.to(device)).squeeze(0)
try:
current_image_features_grid = projected_image_features.view(
num_images, num_height_patches, num_width_patches, embed_dim
)
except RuntimeError as e:
raise RuntimeError(
"Cannot reshape projected_image_features"
f" with shape {projected_image_features.shape} "
f"to ({num_images}, {num_height_patches},"
f" {num_width_patches}, {embed_dim}). "
"Ensure num_projected_patches is compatible"
" with a grid structure. "
f"num_projected_patches={num_projected_patches}, "
f"derived num_height_patches={num_height_patches}. "
) from e
image_newline_expanded = image_newline_emb.expand(
(num_images, num_height_patches, 1, embed_dim)
)
features_with_newlines = torch.cat(
[current_image_features_grid, image_newline_expanded],
dim=2, # Concatenate along width dim
)
new_num_patches_after_newline = num_projected_patches + num_height_patches
features_with_newlines_flat = features_with_newlines.view(
num_images, new_num_patches_after_newline, embed_dim
)
image_new_expanded = image_new_emb.expand((num_images, 1, embed_dim))
final_image_features = torch.cat(
[features_with_newlines_flat, image_new_expanded],
dim=1, # Concatenate along patch sequence dim
)
return final_image_features
def _process_image_pixels(
self,
inputs: TarsierImagePixelInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
pixel_values = inputs["pixel_values"]
image_features_selected = self._image_pixels_to_features(
self.vision_tower, pixel_values
) # type: ignore
if isinstance(image_features_selected, torch.Tensor):
projected_features = self.multi_modal_projector(image_features_selected)
final_features = self._add_tarsier_split_tokens(projected_features)
return final_features
else:
raise TypeError(
f"_image_pixels_to_features type:"
f" {type(image_features_selected)} is not supported"
)
def _process_image_input(
self,
image_input: TarsierImageInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
if image_input["type"] == "image_embeds":
projected_features = image_input["data"]
if isinstance(projected_features, torch.Tensor):
return self._add_tarsier_split_tokens(projected_features)
else:
raise ValueError(
"Incorrect type of image_embeds. "
f"Got type: {type(projected_features)}. "
)
return self._process_image_pixels(image_input)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
return []
return self._process_image_input(image_input)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
hidden_states = self.language_model.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
vllm/transformers_utils/config.py
-1
View File
@@ -123,7 +123,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
qwen3_5_moe="Qwen3_5MoeConfig",
laguna="LagunaConfig",
lfm2_moe="Lfm2MoeConfig",
tarsier2="Tarsier2Config",
**{"unlimited-ocr": "UnlimitedOCRConfig"},
)
vllm/transformers_utils/configs/__init__.py
-2
View File
@@ -86,7 +86,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
"Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
"Tarsier2Config": "vllm.transformers_utils.configs.tarsier2",
# Special case: DeepseekV3Config is from HuggingFace Transformers
"DeepseekV3Config": "transformers",
}
@@ -161,7 +160,6 @@ __all__ = [
"Qwen3_5TextConfig",
"Qwen3_5MoeConfig",
"Qwen3_5MoeTextConfig",
"Tarsier2Config",
]
vllm/transformers_utils/configs/tarsier2.py
-24
View File
@@ -1,24 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import Qwen2VLConfig
class Tarsier2Config(Qwen2VLConfig):
"""
Tarsier2's config.json is written such that AutoConfig.from_pretrained will create
a deeply nested config consisting of:
- LlavaConfig
- Qwen2VLConfig
- Qwen2VLTextConfig
- Qwen2VLVisionConfig
- Qwen2VLConfig
- Qwen2VLTextConfig
- Qwen2VLVisionConfig
When it should really just be a single Qwen2VLConfig.
This class is a hack to stop AutoConfig from creating the nested config structure.
"""
model_type = "tarsier2"