[New model support]Support Tarsier2 (#19887)
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
@@ -32,12 +32,14 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange, repeat
|
||||
from transformers import BatchFeature
|
||||
from transformers import AutoConfig, BatchFeature
|
||||
from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
|
||||
Qwen2VLProcessor)
|
||||
from transformers.models.qwen2_vl.configuration_qwen2_vl import (
|
||||
Qwen2VLConfig, Qwen2VLVisionConfig)
|
||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
|
||||
from transformers.models.qwen2_vl.video_processing_qwen2_vl import (
|
||||
Qwen2VLVideoProcessor)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
|
||||
@@ -69,6 +71,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import uses_mrope
|
||||
from vllm.transformers_utils.processor import (
|
||||
cached_image_processor_from_config)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
|
||||
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
|
||||
SupportsMultiModal, SupportsPP)
|
||||
@@ -1405,3 +1408,87 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||||
connector="visual.merger.",
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
|
||||
class Tarsier2MultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
pass
|
||||
|
||||
|
||||
class Tarsier2ImageProcessor(Qwen2VLImageProcessor):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
size: Optional[dict[str, int]] = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if size is not None and "min_pixels" in size and "max_pixels" in size:
|
||||
# Remap if Tarsier2-specific format is provided
|
||||
remapped_size = {
|
||||
"shortest_edge": size["min_pixels"],
|
||||
"longest_edge": size["max_pixels"]
|
||||
}
|
||||
super().__init__(size=remapped_size, **kwargs)
|
||||
else:
|
||||
super().__init__(size=size, **kwargs)
|
||||
|
||||
|
||||
class Tarsier2Processor(Qwen2VLProcessor):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict,
|
||||
tokenizer: AnyTokenizer,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_processor = Tarsier2ImageProcessor(**vision_config)
|
||||
super().__init__(image_processor=self.image_processor,
|
||||
tokenizer=tokenizer,
|
||||
video_processor=Qwen2VLVideoProcessor(),
|
||||
chat_template=None,
|
||||
**kwargs)
|
||||
|
||||
|
||||
class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo):
|
||||
|
||||
def get_hf_config(self) -> Qwen2VLConfig:
|
||||
model_path = self.ctx.model_config.model
|
||||
original_config = AutoConfig.from_pretrained(model_path)
|
||||
config_dict = original_config.to_dict()
|
||||
correct_config = Qwen2VLConfig.from_dict(config_dict)
|
||||
|
||||
return correct_config
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> Tarsier2Processor:
|
||||
return Tarsier2Processor(
|
||||
vision_config=self.ctx.get_hf_image_processor_config(),
|
||||
tokenizer=self.get_tokenizer(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_image_processor(self) -> Tarsier2ImageProcessor:
|
||||
return Tarsier2ImageProcessor(
|
||||
**self.ctx.get_hf_image_processor_config())
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(Tarsier2MultiModalProcessor,
|
||||
info=Tarsier2ProcessingInfo,
|
||||
dummy_inputs=Qwen2VLDummyInputsBuilder)
|
||||
class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):
|
||||
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
|
||||
"vision_tower.": "visual.",
|
||||
})
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
# Tarsier2 uses llava as model_type, which will create a Qwen2VLConfig
|
||||
# as text_config, we need to reconstruct Qwen2VLConfig from LlavaConfig.
|
||||
config = vllm_config.model_config.hf_config
|
||||
qwen2vl_config = config.text_config
|
||||
qwen2vl_config.architectures = config.architectures
|
||||
vllm_config.model_config.hf_config = qwen2vl_config
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -217,6 +217,7 @@ _MULTIMODAL_MODELS = {
|
||||
"UltravoxModel": ("ultravox", "UltravoxModel"),
|
||||
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
|
||||
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501
|
||||
"Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"), # noqa: E501
|
||||
# [Encoder-decoder]
|
||||
"Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
|
||||
"MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
|
||||
|
||||
Reference in New Issue
Block a user