[New model support]Support Tarsier2 (#19887)

Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
汪志鹏
2025-06-21 04:01:51 +00:00
committed by GitHub
parent 6f170f11dd
commit c3bf9bad11
7 changed files with 152 additions and 1 deletions
+88 -1
View File
@@ -32,12 +32,14 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange, repeat
from transformers import BatchFeature
from transformers import AutoConfig, BatchFeature
from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
Qwen2VLProcessor)
from transformers.models.qwen2_vl.configuration_qwen2_vl import (
Qwen2VLConfig, Qwen2VLVisionConfig)
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from transformers.models.qwen2_vl.video_processing_qwen2_vl import (
Qwen2VLVideoProcessor)
from vllm.config import VllmConfig
from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
@@ -69,6 +71,7 @@ from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.config import uses_mrope
from vllm.transformers_utils.processor import (
cached_image_processor_from_config)
from vllm.transformers_utils.tokenizer import AnyTokenizer
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
SupportsMultiModal, SupportsPP)
@@ -1405,3 +1408,87 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
connector="visual.merger.",
tower_model="visual.",
)
class Tarsier2MultiModalProcessor(Qwen2VLMultiModalProcessor):
pass
class Tarsier2ImageProcessor(Qwen2VLImageProcessor):
def __init__(
self,
size: Optional[dict[str, int]] = None,
**kwargs,
) -> None:
if size is not None and "min_pixels" in size and "max_pixels" in size:
# Remap if Tarsier2-specific format is provided
remapped_size = {
"shortest_edge": size["min_pixels"],
"longest_edge": size["max_pixels"]
}
super().__init__(size=remapped_size, **kwargs)
else:
super().__init__(size=size, **kwargs)
class Tarsier2Processor(Qwen2VLProcessor):
def __init__(
self,
vision_config: dict,
tokenizer: AnyTokenizer,
**kwargs,
):
self.image_processor = Tarsier2ImageProcessor(**vision_config)
super().__init__(image_processor=self.image_processor,
tokenizer=tokenizer,
video_processor=Qwen2VLVideoProcessor(),
chat_template=None,
**kwargs)
class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self) -> Qwen2VLConfig:
model_path = self.ctx.model_config.model
original_config = AutoConfig.from_pretrained(model_path)
config_dict = original_config.to_dict()
correct_config = Qwen2VLConfig.from_dict(config_dict)
return correct_config
def get_hf_processor(self, **kwargs: object) -> Tarsier2Processor:
return Tarsier2Processor(
vision_config=self.ctx.get_hf_image_processor_config(),
tokenizer=self.get_tokenizer(),
**kwargs,
)
def get_image_processor(self) -> Tarsier2ImageProcessor:
return Tarsier2ImageProcessor(
**self.ctx.get_hf_image_processor_config())
@MULTIMODAL_REGISTRY.register_processor(Tarsier2MultiModalProcessor,
info=Tarsier2ProcessingInfo,
dummy_inputs=Qwen2VLDummyInputsBuilder)
class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
"vision_tower.": "visual.",
})
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
# Tarsier2 uses llava as model_type, which will create a Qwen2VLConfig
# as text_config, we need to reconstruct Qwen2VLConfig from LlavaConfig.
config = vllm_config.model_config.hf_config
qwen2vl_config = config.text_config
qwen2vl_config.architectures = config.architectures
vllm_config.model_config.hf_config = qwen2vl_config
super().__init__(vllm_config=vllm_config, prefix=prefix)
def load_weights(self, weights: Iterable[tuple[str,
torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1
View File
@@ -217,6 +217,7 @@ _MULTIMODAL_MODELS = {
"UltravoxModel": ("ultravox", "UltravoxModel"),
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501
"Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"), # noqa: E501
# [Encoder-decoder]
"Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
"MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501