[Model] Remove Tarsier, Tarsier2 (#47143)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -626,8 +626,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
|||||||
| `Step3VLForConditionalGeneration` | Step3-VL | T + I<sup>+</sup> | `stepfun-ai/step3` | | ✅︎ |
|
| `Step3VLForConditionalGeneration` | Step3-VL | T + I<sup>+</sup> | `stepfun-ai/step3` | | ✅︎ |
|
||||||
| `StepVLForConditionalGeneration` | Step3-VL-10B | T + I<sup>+</sup> | `stepfun-ai/Step3-VL-10B` | | ✅︎ |
|
| `StepVLForConditionalGeneration` | Step3-VL-10B | T + I<sup>+</sup> | `stepfun-ai/Step3-VL-10B` | | ✅︎ |
|
||||||
| `Step3p7ForConditionalGeneration` | Step-3.7-Flash | T + I<sup>+</sup> | `stepfun-ai/Step-3.7-Flash` | | ✅︎ |
|
| `Step3p7ForConditionalGeneration` | Step-3.7-Flash | T + I<sup>+</sup> | `stepfun-ai/Step-3.7-Flash` | | ✅︎ |
|
||||||
| `TarsierForConditionalGeneration` | Tarsier | T + I<sup>E+</sup> | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ |
|
|
||||||
| `Tarsier2ForConditionalGeneration`<sup>^</sup> | Tarsier2 | T + I<sup>E+</sup> + V<sup>E+</sup> | `omni-research/Tarsier2-Recap-7b`, `omni-research/Tarsier2-7b-0115` | | ✅︎ |
|
|
||||||
| `UltravoxModel` | Ultravox | T + A<sup>E+</sup> | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ |
|
| `UltravoxModel` | Ultravox | T + A<sup>E+</sup> | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ |
|
||||||
| `UnlimitedOCRForCausalLM` | Unlimited-OCR | T + I<sup>+</sup> | `baidu/Unlimited-OCR`, etc. | ✅︎ | ✅︎ |
|
| `UnlimitedOCRForCausalLM` | Unlimited-OCR | T + I<sup>+</sup> | `baidu/Unlimited-OCR`, etc. | ✅︎ | ✅︎ |
|
||||||
|
|
||||||
|
|||||||
@@ -1275,55 +1275,6 @@ def load_step_vl(question: str, image_urls: list[str]) -> ModelRequestData:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData:
|
|
||||||
model_name = "omni-research/Tarsier-7b"
|
|
||||||
|
|
||||||
engine_args = EngineArgs(
|
|
||||||
model=model_name,
|
|
||||||
trust_remote_code=True,
|
|
||||||
max_model_len=4096,
|
|
||||||
limit_mm_per_prompt={"image": len(image_urls)},
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt = f"USER: {'<image>' * len(image_urls)}\n{question}\n ASSISTANT:"
|
|
||||||
image_data = [fetch_image(url) for url in image_urls]
|
|
||||||
|
|
||||||
return ModelRequestData(
|
|
||||||
engine_args=engine_args,
|
|
||||||
prompt=prompt,
|
|
||||||
image_data=image_data,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
|
|
||||||
model_name = "omni-research/Tarsier2-Recap-7b"
|
|
||||||
|
|
||||||
engine_args = EngineArgs(
|
|
||||||
model=model_name,
|
|
||||||
trust_remote_code=True,
|
|
||||||
max_model_len=32768,
|
|
||||||
limit_mm_per_prompt={"image": len(image_urls)},
|
|
||||||
hf_overrides={
|
|
||||||
"architectures": ["Tarsier2ForConditionalGeneration"],
|
|
||||||
"model_type": "tarsier2",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt = (
|
|
||||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
|
||||||
f"<|im_start|>user\n<|vision_start|>{'<|image_pad|>' * len(image_urls)}"
|
|
||||||
f"<|vision_end|>{question}<|im_end|>\n"
|
|
||||||
"<|im_start|>assistant\n"
|
|
||||||
)
|
|
||||||
image_data = [fetch_image(url) for url in image_urls]
|
|
||||||
|
|
||||||
return ModelRequestData(
|
|
||||||
engine_args=engine_args,
|
|
||||||
prompt=prompt,
|
|
||||||
image_data=image_data,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# GLM-4.1V
|
# GLM-4.1V
|
||||||
def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData:
|
def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||||
model_name = "zai-org/GLM-4.1V-9B-Thinking"
|
model_name = "zai-org/GLM-4.1V-9B-Thinking"
|
||||||
@@ -1507,8 +1458,6 @@ model_example_map = {
|
|||||||
"smolvlm": load_smolvlm,
|
"smolvlm": load_smolvlm,
|
||||||
"step3": load_step3,
|
"step3": load_step3,
|
||||||
"stepvl": load_step_vl,
|
"stepvl": load_step_vl,
|
||||||
"tarsier": load_tarsier,
|
|
||||||
"tarsier2": load_tarsier2,
|
|
||||||
"glm4_1v": load_glm4_1v,
|
"glm4_1v": load_glm4_1v,
|
||||||
"glm4_5v": load_glm4_5v,
|
"glm4_5v": load_glm4_5v,
|
||||||
"glm4_5v_fp8": load_glm4_5v_fp8,
|
"glm4_5v_fp8": load_glm4_5v_fp8,
|
||||||
|
|||||||
@@ -2347,65 +2347,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# omni-research/Tarsier-7b
|
|
||||||
def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
|
|
||||||
assert modality == "image"
|
|
||||||
model_name = "omni-research/Tarsier-7b"
|
|
||||||
|
|
||||||
engine_args = EngineArgs(
|
|
||||||
model=model_name,
|
|
||||||
trust_remote_code=True,
|
|
||||||
max_model_len=4096,
|
|
||||||
limit_mm_per_prompt={modality: 1},
|
|
||||||
)
|
|
||||||
prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
|
|
||||||
|
|
||||||
return ModelRequestData(
|
|
||||||
engine_args=engine_args,
|
|
||||||
prompts=prompts,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
|
|
||||||
model_name = "omni-research/Tarsier2-Recap-7b"
|
|
||||||
|
|
||||||
mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
|
|
||||||
engine_args = EngineArgs(
|
|
||||||
model=model_name,
|
|
||||||
max_model_len=4096,
|
|
||||||
hf_overrides={
|
|
||||||
"architectures": ["Tarsier2ForConditionalGeneration"],
|
|
||||||
"model_type": "tarsier2",
|
|
||||||
},
|
|
||||||
limit_mm_per_prompt=mm_limit,
|
|
||||||
)
|
|
||||||
|
|
||||||
image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
|
||||||
video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
|
|
||||||
|
|
||||||
if modality == "image":
|
|
||||||
placeholder = image_placeholder
|
|
||||||
elif modality == "video":
|
|
||||||
placeholder = video_placeholder
|
|
||||||
elif modality == "image+video":
|
|
||||||
placeholder = image_placeholder + video_placeholder
|
|
||||||
|
|
||||||
prompts = [
|
|
||||||
(
|
|
||||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
|
||||||
f"<|im_start|>user\n{placeholder}"
|
|
||||||
f"{question}<|im_end|>\n"
|
|
||||||
"<|im_start|>assistant\n"
|
|
||||||
)
|
|
||||||
for question in questions
|
|
||||||
]
|
|
||||||
|
|
||||||
return ModelRequestData(
|
|
||||||
engine_args=engine_args,
|
|
||||||
prompts=prompts,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
model_example_map = {
|
model_example_map = {
|
||||||
"aria": run_aria,
|
"aria": run_aria,
|
||||||
"aya_vision": run_aya_vision,
|
"aya_vision": run_aya_vision,
|
||||||
@@ -2479,8 +2420,6 @@ model_example_map = {
|
|||||||
"smolvlm": run_smolvlm,
|
"smolvlm": run_smolvlm,
|
||||||
"step3": run_step3,
|
"step3": run_step3,
|
||||||
"stepvl": run_step_vl,
|
"stepvl": run_step_vl,
|
||||||
"tarsier": run_tarsier,
|
|
||||||
"tarsier2": run_tarsier2,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -993,31 +993,6 @@ VLM_TEST_SETTINGS = {
|
|||||||
hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,
|
hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,
|
||||||
num_logprobs=10,
|
num_logprobs=10,
|
||||||
),
|
),
|
||||||
"tarsier": VLMTestInfo(
|
|
||||||
models=["omni-research/Tarsier-7b"],
|
|
||||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
|
||||||
prompt_formatter=lambda img_prompt: f"USER: {img_prompt} ASSISTANT:",
|
|
||||||
max_model_len=4096,
|
|
||||||
max_num_seqs=2,
|
|
||||||
auto_cls=AutoModelForImageTextToText,
|
|
||||||
patch_hf_runner=model_utils.tarsier_patch_hf_runner,
|
|
||||||
),
|
|
||||||
"tarsier2": VLMTestInfo(
|
|
||||||
models=["omni-research/Tarsier2-Recap-7b"],
|
|
||||||
test_type=(
|
|
||||||
VLMTestType.IMAGE,
|
|
||||||
VLMTestType.MULTI_IMAGE,
|
|
||||||
VLMTestType.VIDEO,
|
|
||||||
),
|
|
||||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
|
||||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
|
|
||||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
|
|
||||||
max_model_len=4096,
|
|
||||||
max_num_seqs=2,
|
|
||||||
auto_cls=AutoModelForImageTextToText,
|
|
||||||
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
|
|
||||||
marks=[pytest.mark.skip("Model initialization hangs")],
|
|
||||||
),
|
|
||||||
### Tensor parallel / multi-gpu broadcast tests
|
### Tensor parallel / multi-gpu broadcast tests
|
||||||
"chameleon-broadcast": VLMTestInfo(
|
"chameleon-broadcast": VLMTestInfo(
|
||||||
models=["facebook/chameleon-7b"],
|
models=["facebook/chameleon-7b"],
|
||||||
|
|||||||
@@ -1220,18 +1220,6 @@ def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
|||||||
return hf_model
|
return hf_model
|
||||||
|
|
||||||
|
|
||||||
def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
|
||||||
from vllm.model_executor.models.tarsier import get_vision_encoder_info
|
|
||||||
|
|
||||||
vision_encoder_info = get_vision_encoder_info(hf_model.config)
|
|
||||||
|
|
||||||
hf_processor = hf_model.processor
|
|
||||||
if hf_processor.patch_size is None:
|
|
||||||
hf_processor.patch_size = vision_encoder_info.get_patch_size()
|
|
||||||
|
|
||||||
return hf_model
|
|
||||||
|
|
||||||
|
|
||||||
def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
||||||
"""Patch HfRunner for Voxtral's conversation-based processor.
|
"""Patch HfRunner for Voxtral's conversation-based processor.
|
||||||
|
|
||||||
|
|||||||
@@ -1369,22 +1369,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
|||||||
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
|
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
),
|
),
|
||||||
"TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
|
|
||||||
"Tarsier2ForConditionalGeneration": _HfExamplesInfo(
|
|
||||||
"omni-research/Tarsier2-Recap-7b",
|
|
||||||
hf_overrides={
|
|
||||||
"architectures": ["Tarsier2ForConditionalGeneration"],
|
|
||||||
"model_type": "tarsier2",
|
|
||||||
},
|
|
||||||
max_transformers_version="5.3",
|
|
||||||
transformers_version_reason={
|
|
||||||
"vllm": (
|
|
||||||
"Qwen2VLConfig was split into Qwen2VLConfig + "
|
|
||||||
"Qwen2VLTextConfig in transformers v5, breaking "
|
|
||||||
"attribute access (num_attention_heads, hidden_size, etc.)"
|
|
||||||
)
|
|
||||||
},
|
|
||||||
),
|
|
||||||
"VoxtralForConditionalGeneration": _HfExamplesInfo(
|
"VoxtralForConditionalGeneration": _HfExamplesInfo(
|
||||||
"mistralai/Voxtral-Mini-3B-2507",
|
"mistralai/Voxtral-Mini-3B-2507",
|
||||||
tokenizer_mode="mistral",
|
tokenizer_mode="mistral",
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (
|
|||||||
Qwen2VLVisionConfig,
|
Qwen2VLVisionConfig,
|
||||||
)
|
)
|
||||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
|
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
|
||||||
from transformers.models.qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor
|
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
@@ -86,7 +85,6 @@ from vllm.multimodal.processing import (
|
|||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tokenizers import TokenizerLike
|
|
||||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||||
@@ -1755,83 +1753,3 @@ class Qwen2VLForConditionalGeneration(
|
|||||||
vision_config = hf_config.vision_config
|
vision_config = hf_config.vision_config
|
||||||
merge_size = vision_config.spatial_merge_size
|
merge_size = vision_config.spatial_merge_size
|
||||||
return num_vision_tokens // merge_size**2
|
return num_vision_tokens // merge_size**2
|
||||||
|
|
||||||
|
|
||||||
class Tarsier2MultiModalProcessor(Qwen2VLMultiModalProcessor):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class Tarsier2ImageProcessor(Qwen2VLImageProcessor):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
size: dict[str, int] | None = None,
|
|
||||||
**kwargs,
|
|
||||||
) -> None:
|
|
||||||
if size is not None and "min_pixels" in size and "max_pixels" in size:
|
|
||||||
# Remap if Tarsier2-specific format is provided
|
|
||||||
remapped_size = {
|
|
||||||
"shortest_edge": size["min_pixels"],
|
|
||||||
"longest_edge": size["max_pixels"],
|
|
||||||
}
|
|
||||||
super().__init__(size=remapped_size, **kwargs)
|
|
||||||
else:
|
|
||||||
super().__init__(size=size, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
class Tarsier2Processor(Qwen2VLProcessor):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
image_processor: Tarsier2ImageProcessor,
|
|
||||||
tokenizer: TokenizerLike,
|
|
||||||
video_processor: Qwen2VLVideoProcessor,
|
|
||||||
**kwargs,
|
|
||||||
):
|
|
||||||
super().__init__(
|
|
||||||
image_processor=image_processor,
|
|
||||||
tokenizer=tokenizer,
|
|
||||||
video_processor=video_processor,
|
|
||||||
chat_template=None,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo):
|
|
||||||
def get_hf_config(self) -> Qwen2VLConfig:
|
|
||||||
model_path = self.ctx.model_config.model
|
|
||||||
correct_config = Qwen2VLConfig.from_pretrained(model_path)
|
|
||||||
|
|
||||||
return correct_config
|
|
||||||
|
|
||||||
def get_hf_processor(self, **kwargs: object) -> Tarsier2Processor:
|
|
||||||
vision_config = self.ctx.get_hf_image_processor_config()
|
|
||||||
image_processor = Tarsier2ImageProcessor(**vision_config)
|
|
||||||
video_processor = Qwen2VLVideoProcessor(**vision_config)
|
|
||||||
return Tarsier2Processor(
|
|
||||||
image_processor=image_processor,
|
|
||||||
video_processor=video_processor,
|
|
||||||
tokenizer=self.get_tokenizer(),
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_image_processor(self) -> Tarsier2ImageProcessor:
|
|
||||||
return Tarsier2ImageProcessor(**self.ctx.get_hf_image_processor_config())
|
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
|
||||||
Tarsier2MultiModalProcessor,
|
|
||||||
info=Tarsier2ProcessingInfo,
|
|
||||||
dummy_inputs=Qwen2VLDummyInputsBuilder,
|
|
||||||
)
|
|
||||||
class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):
|
|
||||||
hf_to_vllm_mapper = WeightsMapper(
|
|
||||||
orig_to_new_prefix={
|
|
||||||
"vision_tower.": "visual.",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
||||||
skip_prefixes = []
|
|
||||||
if self.visual is None:
|
|
||||||
skip_prefixes.extend(["visual."])
|
|
||||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
|
||||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
|
||||||
|
|||||||
@@ -571,11 +571,6 @@ _MULTIMODAL_MODELS = {
|
|||||||
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
|
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
|
||||||
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
|
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
|
||||||
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
|
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
|
||||||
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
|
|
||||||
"Tarsier2ForConditionalGeneration": (
|
|
||||||
"qwen2_vl",
|
|
||||||
"Tarsier2ForConditionalGeneration",
|
|
||||||
),
|
|
||||||
"UltravoxModel": ("ultravox", "UltravoxModel"),
|
"UltravoxModel": ("ultravox", "UltravoxModel"),
|
||||||
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
|
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
|
||||||
"VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
|
"VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
|
||||||
@@ -736,6 +731,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
|
|||||||
"AquilaForCausalLM": "0.24.0",
|
"AquilaForCausalLM": "0.24.0",
|
||||||
"Grok1ModelForCausalLM": "0.24.0",
|
"Grok1ModelForCausalLM": "0.24.0",
|
||||||
"Grok1ForCausalLM": "0.24.0",
|
"Grok1ForCausalLM": "0.24.0",
|
||||||
|
"TarsierForConditionalGeneration": "0.24.0",
|
||||||
|
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
|
||||||
}
|
}
|
||||||
|
|
||||||
_OOT_SUPPORTED_MODELS = {
|
_OOT_SUPPORTED_MODELS = {
|
||||||
|
|||||||
@@ -1,591 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import math
|
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
|
||||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
|
|
||||||
|
|
||||||
import torch
|
|
||||||
import torch.nn as nn
|
|
||||||
from transformers import (
|
|
||||||
BatchFeature,
|
|
||||||
CLIPVisionConfig,
|
|
||||||
PretrainedConfig,
|
|
||||||
SiglipVisionConfig,
|
|
||||||
)
|
|
||||||
from transformers import LlavaConfig as HfLlavaConfig
|
|
||||||
from transformers.image_utils import ImageInput, get_image_size, to_numpy_array
|
|
||||||
from transformers.models.llava import LlavaProcessor
|
|
||||||
from transformers.processing_utils import ProcessingKwargs, Unpack
|
|
||||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
|
||||||
from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
|
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
|
||||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
|
||||||
from vllm.multimodal.parse import (
|
|
||||||
ImageEmbeddingItems,
|
|
||||||
ImageProcessorItems,
|
|
||||||
ImageSize,
|
|
||||||
MultiModalDataItems,
|
|
||||||
)
|
|
||||||
from vllm.multimodal.processing import (
|
|
||||||
BaseMultiModalProcessor,
|
|
||||||
BaseProcessingInfo,
|
|
||||||
PromptReplacement,
|
|
||||||
PromptUpdate,
|
|
||||||
)
|
|
||||||
from vllm.sequence import IntermediateTensors
|
|
||||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
|
||||||
|
|
||||||
from .clip import CLIPVisionModel
|
|
||||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
|
||||||
from .siglip import SiglipVisionModel
|
|
||||||
from .utils import (
|
|
||||||
AutoWeightsLoader,
|
|
||||||
get_layer_index,
|
|
||||||
init_vllm_registered_model,
|
|
||||||
maybe_prefix,
|
|
||||||
)
|
|
||||||
from .vision import (
|
|
||||||
VisionEncoderInfo,
|
|
||||||
get_num_selected_vision_tokens,
|
|
||||||
get_vision_encoder_info,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierImagePixelInputs(TensorSchema):
|
|
||||||
"""
|
|
||||||
Dimensions:
|
|
||||||
- bn: Batch size * number of images
|
|
||||||
- c: Number of channels (3)
|
|
||||||
- h: Height
|
|
||||||
- w: Width
|
|
||||||
"""
|
|
||||||
|
|
||||||
type: Literal["pixel_values"] = "pixel_values"
|
|
||||||
pixel_values: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierImageEmbeddingInputs(TensorSchema):
|
|
||||||
"""
|
|
||||||
Dimensions:
|
|
||||||
- bn: Batch size * number of images
|
|
||||||
- ifs: Image feature size
|
|
||||||
- hs: Hidden size (must match the hidden size of language model
|
|
||||||
backbone)
|
|
||||||
"""
|
|
||||||
|
|
||||||
type: Literal["image_embeds"] = "image_embeds"
|
|
||||||
data: Annotated[torch.Tensor, TensorShape("bn", "ifs", "hs")]
|
|
||||||
|
|
||||||
|
|
||||||
TarsierImageInputs: TypeAlias = TarsierImagePixelInputs | TarsierImageEmbeddingInputs
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierHfConfig(Protocol): # Based on the Tarsier's LlavaConfig
|
|
||||||
vision_config: Final[PretrainedConfig]
|
|
||||||
text_config: Final[PretrainedConfig] # Added from Tarsier's LlavaConfig
|
|
||||||
image_token_index: Final[int]
|
|
||||||
vision_feature_select_strategy: Final[str]
|
|
||||||
vision_feature_layer: Final[int | list[int]]
|
|
||||||
projector_hidden_act: Final[str]
|
|
||||||
image_newline_idx: Final[int]
|
|
||||||
image_new_idx: Final[int]
|
|
||||||
multimodal_projector_bias: bool = True
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierProcessorKwargs(ProcessingKwargs, total=False):
|
|
||||||
_defaults = {
|
|
||||||
"text_kwargs": {
|
|
||||||
"padding": False,
|
|
||||||
},
|
|
||||||
"images_kwargs": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierProcessor(LlavaProcessor):
|
|
||||||
def __call__(
|
|
||||||
self,
|
|
||||||
images: ImageInput = None,
|
|
||||||
text: TextInput
|
|
||||||
| PreTokenizedInput
|
|
||||||
| list[TextInput]
|
|
||||||
| list[PreTokenizedInput] = None,
|
|
||||||
audio=None,
|
|
||||||
videos=None,
|
|
||||||
**kwargs: Unpack[TarsierProcessorKwargs],
|
|
||||||
) -> BatchFeature:
|
|
||||||
if images is None and text is None:
|
|
||||||
raise ValueError("You have to specify at least one of `images` or `text`.")
|
|
||||||
|
|
||||||
output_kwargs = self._merge_kwargs(
|
|
||||||
TarsierProcessorKwargs,
|
|
||||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
if images is not None:
|
|
||||||
image_inputs = self.image_processor(
|
|
||||||
images, **output_kwargs["images_kwargs"]
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
image_inputs = {}
|
|
||||||
|
|
||||||
if isinstance(text, str):
|
|
||||||
text = [text]
|
|
||||||
elif not isinstance(text, list) and not isinstance(text[0], str):
|
|
||||||
raise ValueError(
|
|
||||||
"Invalid input text. Please provide a string, or a list of strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
# try to expand inputs in processing if we have the necessary parts
|
|
||||||
prompt_strings = text
|
|
||||||
if image_inputs.get("pixel_values") is not None:
|
|
||||||
# Replace the image token with the expanded image token sequence
|
|
||||||
pixel_values = image_inputs["pixel_values"]
|
|
||||||
height, width = get_image_size(to_numpy_array(pixel_values[0]))
|
|
||||||
num_image_tokens = (
|
|
||||||
(height // self.patch_size) * (width // self.patch_size + 1)
|
|
||||||
+ self.num_additional_image_tokens
|
|
||||||
+ 1
|
|
||||||
)
|
|
||||||
if self.vision_feature_select_strategy == "default":
|
|
||||||
num_image_tokens -= 1
|
|
||||||
|
|
||||||
prompt_strings = []
|
|
||||||
for sample in text:
|
|
||||||
sample = sample.replace(
|
|
||||||
self.image_token, self.image_token * num_image_tokens
|
|
||||||
)
|
|
||||||
prompt_strings.append(sample)
|
|
||||||
|
|
||||||
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
|
||||||
text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"])
|
|
||||||
return BatchFeature(
|
|
||||||
data={**text_inputs, **image_inputs}, tensor_type=return_tensors
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierMultiModalProjector(nn.Module):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
vision_hidden_size: int,
|
|
||||||
text_hidden_size: int,
|
|
||||||
projector_hidden_act: str,
|
|
||||||
multimodal_projector_bias: bool,
|
|
||||||
quant_config: QuantizationConfig | None = None,
|
|
||||||
prefix: str = "",
|
|
||||||
):
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
self.linear_1 = ColumnParallelLinear(
|
|
||||||
vision_hidden_size,
|
|
||||||
text_hidden_size,
|
|
||||||
bias=multimodal_projector_bias,
|
|
||||||
quant_config=quant_config,
|
|
||||||
prefix=f"{prefix}.linear_1",
|
|
||||||
)
|
|
||||||
self.act = get_act_fn(projector_hidden_act)
|
|
||||||
self.linear_2 = RowParallelLinear(
|
|
||||||
text_hidden_size,
|
|
||||||
text_hidden_size,
|
|
||||||
bias=multimodal_projector_bias,
|
|
||||||
quant_config=quant_config,
|
|
||||||
prefix=f"{prefix}.linear_2",
|
|
||||||
)
|
|
||||||
|
|
||||||
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
|
|
||||||
hidden_states, _ = self.linear_1(image_features)
|
|
||||||
hidden_states = self.act(hidden_states)
|
|
||||||
hidden_states, _ = self.linear_2(hidden_states)
|
|
||||||
return hidden_states
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierProcessingInfo(BaseProcessingInfo):
|
|
||||||
def get_hf_config(self) -> TarsierHfConfig:
|
|
||||||
return self.ctx.get_hf_config(HfLlavaConfig)
|
|
||||||
|
|
||||||
def get_vision_encoder_info(self) -> VisionEncoderInfo:
|
|
||||||
return get_vision_encoder_info(self.get_hf_config())
|
|
||||||
|
|
||||||
def get_hf_processor(self, **kwargs: object) -> TarsierProcessor:
|
|
||||||
vision_info = self.get_vision_encoder_info()
|
|
||||||
|
|
||||||
kwargs.setdefault("patch_size", vision_info.get_patch_size())
|
|
||||||
|
|
||||||
return self.ctx.get_hf_processor(TarsierProcessor, **kwargs)
|
|
||||||
|
|
||||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
|
||||||
return {"image": None}
|
|
||||||
|
|
||||||
def get_num_image_tokens(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
image_width: int,
|
|
||||||
image_height: int,
|
|
||||||
) -> int:
|
|
||||||
hf_config = self.get_hf_config()
|
|
||||||
vision_encoder_info = self.get_vision_encoder_info()
|
|
||||||
num_projected_patches = get_num_selected_vision_tokens(
|
|
||||||
vision_encoder_info.get_num_image_tokens(
|
|
||||||
image_width=image_width,
|
|
||||||
image_height=image_height,
|
|
||||||
),
|
|
||||||
hf_config.vision_feature_select_strategy,
|
|
||||||
)
|
|
||||||
if num_projected_patches <= 0:
|
|
||||||
default_size = self.get_image_size_with_most_features()
|
|
||||||
num_projected_patches_default = get_num_selected_vision_tokens(
|
|
||||||
vision_encoder_info.get_num_image_tokens(
|
|
||||||
image_width=default_size.width,
|
|
||||||
image_height=default_size.height,
|
|
||||||
),
|
|
||||||
hf_config.vision_feature_select_strategy,
|
|
||||||
)
|
|
||||||
if num_projected_patches_default <= 0:
|
|
||||||
raise ValueError("Could not determine a valid number of image patches.")
|
|
||||||
num_projected_patches = num_projected_patches_default
|
|
||||||
num_height_patches = int(math.sqrt(num_projected_patches))
|
|
||||||
total_image_tokens_for_llm = num_projected_patches + num_height_patches + 1
|
|
||||||
return total_image_tokens_for_llm
|
|
||||||
|
|
||||||
def get_image_size_with_most_features(self) -> ImageSize:
|
|
||||||
vision_encoder_info = self.get_vision_encoder_info()
|
|
||||||
width = height = vision_encoder_info.get_image_size()
|
|
||||||
return ImageSize(width=width, height=height)
|
|
||||||
|
|
||||||
def get_max_image_tokens(self) -> int:
|
|
||||||
target_width, target_height = self.get_image_size_with_most_features()
|
|
||||||
return self.get_num_image_tokens(
|
|
||||||
image_width=target_width,
|
|
||||||
image_height=target_height,
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_image_newline_idx(self) -> int:
|
|
||||||
return self.get_hf_config().image_newline_idx
|
|
||||||
|
|
||||||
def get_image_new_idx(self) -> int:
|
|
||||||
return self.get_hf_config().image_new_idx
|
|
||||||
|
|
||||||
|
|
||||||
_I_Tarsier = TypeVar("_I_Tarsier", bound=TarsierProcessingInfo)
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierDummyInputsBuilder(LlavaDummyInputsBuilder[_I_Tarsier]):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class TarsierMultiModalProcessor(BaseMultiModalProcessor[_I_Tarsier]):
|
|
||||||
def _get_mm_fields_config(
|
|
||||||
self,
|
|
||||||
hf_inputs: BatchFeature,
|
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
|
||||||
) -> Mapping[str, MultiModalFieldConfig]:
|
|
||||||
return dict(
|
|
||||||
pixel_values=MultiModalFieldConfig.batched("image"),
|
|
||||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
|
||||||
self,
|
|
||||||
mm_items: MultiModalDataItems,
|
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
|
||||||
out_mm_kwargs: MultiModalKwargsItems,
|
|
||||||
) -> Sequence[PromptUpdate]:
|
|
||||||
hf_config = self.info.get_hf_config()
|
|
||||||
image_token_id = hf_config.image_token_index # The <IMAGE> token ID
|
|
||||||
|
|
||||||
def get_replacement(item_idx: int):
|
|
||||||
images = mm_items.get_items(
|
|
||||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
|
||||||
)
|
|
||||||
|
|
||||||
if isinstance(images, ImageEmbeddingItems):
|
|
||||||
num_projected_patches = images.get_feature_size(item_idx)
|
|
||||||
# This assumes num_projected_patches is a perfect square
|
|
||||||
num_height_patches = int(math.sqrt(num_projected_patches))
|
|
||||||
num_final_image_tokens = num_projected_patches + num_height_patches + 1
|
|
||||||
else:
|
|
||||||
image_size = images.get_image_size(item_idx)
|
|
||||||
num_final_image_tokens = self.info.get_num_image_tokens(
|
|
||||||
image_width=image_size.width,
|
|
||||||
image_height=image_size.height,
|
|
||||||
)
|
|
||||||
|
|
||||||
return [image_token_id] * num_final_image_tokens
|
|
||||||
|
|
||||||
return [
|
|
||||||
PromptReplacement(
|
|
||||||
modality="image",
|
|
||||||
target=[image_token_id], # Replace each single <IMAGE> token
|
|
||||||
replacement=get_replacement,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def init_vision_tower_for_tarsier(
|
|
||||||
hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol
|
|
||||||
quant_config: QuantizationConfig | None,
|
|
||||||
*,
|
|
||||||
require_post_norm: bool | None = None,
|
|
||||||
prefix: str = "",
|
|
||||||
) -> CLIPVisionModel | SiglipVisionModel:
|
|
||||||
vision_config = hf_config.vision_config
|
|
||||||
|
|
||||||
feature_layers = hf_config.vision_feature_layer
|
|
||||||
base_num_hidden_layers = vision_config.num_hidden_layers
|
|
||||||
|
|
||||||
if isinstance(feature_layers, int):
|
|
||||||
num_hidden_layers_to_init = get_layer_index(
|
|
||||||
feature_layers, base_num_hidden_layers
|
|
||||||
)
|
|
||||||
elif isinstance(feature_layers, (list, tuple)):
|
|
||||||
num_hidden_layers_to_init = max(
|
|
||||||
get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise TypeError(
|
|
||||||
f"vision_layer_feature type: {type(feature_layers)} is not supported"
|
|
||||||
)
|
|
||||||
|
|
||||||
if isinstance(vision_config, CLIPVisionConfig):
|
|
||||||
return CLIPVisionModel(
|
|
||||||
vision_config,
|
|
||||||
quant_config=quant_config,
|
|
||||||
num_hidden_layers_override=num_hidden_layers_to_init,
|
|
||||||
require_post_norm=require_post_norm,
|
|
||||||
prefix=prefix,
|
|
||||||
)
|
|
||||||
elif isinstance(vision_config, SiglipVisionConfig):
|
|
||||||
return SiglipVisionModel(
|
|
||||||
vision_config,
|
|
||||||
quant_config=quant_config,
|
|
||||||
num_hidden_layers_override=num_hidden_layers_to_init,
|
|
||||||
require_post_norm=require_post_norm,
|
|
||||||
prefix=prefix,
|
|
||||||
)
|
|
||||||
|
|
||||||
msg = f"Unsupported vision config for Tarsier: {type(vision_config)}"
|
|
||||||
raise NotImplementedError(msg)
|
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
|
||||||
TarsierMultiModalProcessor,
|
|
||||||
info=TarsierProcessingInfo,
|
|
||||||
dummy_inputs=TarsierDummyInputsBuilder,
|
|
||||||
)
|
|
||||||
class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
|
|
||||||
packed_modules_mapping = {
|
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
|
||||||
}
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
|
||||||
if modality.startswith("image"):
|
|
||||||
return "<image>"
|
|
||||||
|
|
||||||
raise ValueError("Only image modality is supported")
|
|
||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
config: TarsierHfConfig = vllm_config.model_config.hf_config
|
|
||||||
quant_config = vllm_config.quant_config
|
|
||||||
|
|
||||||
self.config = config # Storing the Tarsier-specific HF config
|
|
||||||
|
|
||||||
with self._mark_tower_model(vllm_config, "image"):
|
|
||||||
self.vision_tower = init_vision_tower_for_tarsier(
|
|
||||||
config,
|
|
||||||
quant_config=quant_config,
|
|
||||||
require_post_norm=False,
|
|
||||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
|
||||||
)
|
|
||||||
projector_bias = getattr(config, "multimodal_projector_bias", True)
|
|
||||||
|
|
||||||
self.multi_modal_projector = TarsierMultiModalProjector(
|
|
||||||
vision_hidden_size=config.vision_config.hidden_size,
|
|
||||||
text_hidden_size=config.text_config.hidden_size,
|
|
||||||
projector_hidden_act=config.projector_hidden_act,
|
|
||||||
multimodal_projector_bias=projector_bias,
|
|
||||||
quant_config=quant_config,
|
|
||||||
prefix=maybe_prefix(prefix, "multi_modal_projector"),
|
|
||||||
)
|
|
||||||
self.register_buffer(
|
|
||||||
"image_newline_idx_tensor",
|
|
||||||
torch.tensor([config.image_newline_idx], dtype=torch.long),
|
|
||||||
persistent=False,
|
|
||||||
)
|
|
||||||
self.register_buffer(
|
|
||||||
"image_new_idx_tensor",
|
|
||||||
torch.tensor([config.image_new_idx], dtype=torch.long),
|
|
||||||
persistent=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
with self._mark_language_model(vllm_config):
|
|
||||||
self.language_model = init_vllm_registered_model(
|
|
||||||
vllm_config=vllm_config,
|
|
||||||
# Use text_config from Tarsier's main config
|
|
||||||
hf_config=config.text_config,
|
|
||||||
prefix=maybe_prefix(prefix, "language_model"),
|
|
||||||
)
|
|
||||||
|
|
||||||
self.make_empty_intermediate_tensors = (
|
|
||||||
self.language_model.make_empty_intermediate_tensors
|
|
||||||
)
|
|
||||||
|
|
||||||
def _parse_and_validate_image_input(
|
|
||||||
self, **kwargs: object
|
|
||||||
) -> TarsierImageInputs | None:
|
|
||||||
pixel_values = kwargs.pop("pixel_values", None)
|
|
||||||
image_embeds = kwargs.pop("image_embeds", None)
|
|
||||||
|
|
||||||
if pixel_values is None and image_embeds is None:
|
|
||||||
return None
|
|
||||||
|
|
||||||
if pixel_values is not None:
|
|
||||||
return TarsierImagePixelInputs(
|
|
||||||
type="pixel_values",
|
|
||||||
pixel_values=pixel_values,
|
|
||||||
)
|
|
||||||
|
|
||||||
if image_embeds is not None:
|
|
||||||
return TarsierImageEmbeddingInputs(
|
|
||||||
type="image_embeds",
|
|
||||||
data=image_embeds,
|
|
||||||
)
|
|
||||||
|
|
||||||
raise AssertionError("This line should be unreachable.")
|
|
||||||
|
|
||||||
def _image_pixels_to_features(
|
|
||||||
self,
|
|
||||||
vision_tower: CLIPVisionModel | SiglipVisionModel,
|
|
||||||
pixel_values: torch.Tensor | list[torch.Tensor],
|
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
|
||||||
# From vLLM LLaVA, vision tower output handling
|
|
||||||
return vision_tower(
|
|
||||||
pixel_values,
|
|
||||||
feature_select_strategy=self.config.vision_feature_select_strategy,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _add_tarsier_split_tokens(
|
|
||||||
self, projected_image_features: torch.Tensor
|
|
||||||
) -> torch.Tensor:
|
|
||||||
"""
|
|
||||||
Implements Tarsier's `add_split_tokens` logic.
|
|
||||||
"""
|
|
||||||
num_images, num_projected_patches, embed_dim = projected_image_features.shape
|
|
||||||
num_height_patches = int(math.sqrt(num_projected_patches))
|
|
||||||
num_width_patches = num_projected_patches // num_height_patches
|
|
||||||
device = projected_image_features.device
|
|
||||||
embedding_layer = self.language_model.model.embed_tokens
|
|
||||||
image_newline_emb = embedding_layer(
|
|
||||||
self.image_newline_idx_tensor.to(device)
|
|
||||||
).squeeze(0)
|
|
||||||
image_new_emb = embedding_layer(self.image_new_idx_tensor.to(device)).squeeze(0)
|
|
||||||
try:
|
|
||||||
current_image_features_grid = projected_image_features.view(
|
|
||||||
num_images, num_height_patches, num_width_patches, embed_dim
|
|
||||||
)
|
|
||||||
except RuntimeError as e:
|
|
||||||
raise RuntimeError(
|
|
||||||
"Cannot reshape projected_image_features"
|
|
||||||
f" with shape {projected_image_features.shape} "
|
|
||||||
f"to ({num_images}, {num_height_patches},"
|
|
||||||
f" {num_width_patches}, {embed_dim}). "
|
|
||||||
"Ensure num_projected_patches is compatible"
|
|
||||||
" with a grid structure. "
|
|
||||||
f"num_projected_patches={num_projected_patches}, "
|
|
||||||
f"derived num_height_patches={num_height_patches}. "
|
|
||||||
) from e
|
|
||||||
|
|
||||||
image_newline_expanded = image_newline_emb.expand(
|
|
||||||
(num_images, num_height_patches, 1, embed_dim)
|
|
||||||
)
|
|
||||||
features_with_newlines = torch.cat(
|
|
||||||
[current_image_features_grid, image_newline_expanded],
|
|
||||||
dim=2, # Concatenate along width dim
|
|
||||||
)
|
|
||||||
new_num_patches_after_newline = num_projected_patches + num_height_patches
|
|
||||||
features_with_newlines_flat = features_with_newlines.view(
|
|
||||||
num_images, new_num_patches_after_newline, embed_dim
|
|
||||||
)
|
|
||||||
image_new_expanded = image_new_emb.expand((num_images, 1, embed_dim))
|
|
||||||
final_image_features = torch.cat(
|
|
||||||
[features_with_newlines_flat, image_new_expanded],
|
|
||||||
dim=1, # Concatenate along patch sequence dim
|
|
||||||
)
|
|
||||||
return final_image_features
|
|
||||||
|
|
||||||
def _process_image_pixels(
|
|
||||||
self,
|
|
||||||
inputs: TarsierImagePixelInputs,
|
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
|
||||||
pixel_values = inputs["pixel_values"]
|
|
||||||
image_features_selected = self._image_pixels_to_features(
|
|
||||||
self.vision_tower, pixel_values
|
|
||||||
) # type: ignore
|
|
||||||
if isinstance(image_features_selected, torch.Tensor):
|
|
||||||
projected_features = self.multi_modal_projector(image_features_selected)
|
|
||||||
final_features = self._add_tarsier_split_tokens(projected_features)
|
|
||||||
return final_features
|
|
||||||
else:
|
|
||||||
raise TypeError(
|
|
||||||
f"_image_pixels_to_features type:"
|
|
||||||
f" {type(image_features_selected)} is not supported"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _process_image_input(
|
|
||||||
self,
|
|
||||||
image_input: TarsierImageInputs,
|
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
|
||||||
if image_input["type"] == "image_embeds":
|
|
||||||
projected_features = image_input["data"]
|
|
||||||
if isinstance(projected_features, torch.Tensor):
|
|
||||||
return self._add_tarsier_split_tokens(projected_features)
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
"Incorrect type of image_embeds. "
|
|
||||||
f"Got type: {type(projected_features)}. "
|
|
||||||
)
|
|
||||||
|
|
||||||
return self._process_image_pixels(image_input)
|
|
||||||
|
|
||||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
|
||||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
|
||||||
if image_input is None:
|
|
||||||
return []
|
|
||||||
return self._process_image_input(image_input)
|
|
||||||
|
|
||||||
def forward(
|
|
||||||
self,
|
|
||||||
input_ids: torch.Tensor | None,
|
|
||||||
positions: torch.Tensor,
|
|
||||||
intermediate_tensors: IntermediateTensors | None = None,
|
|
||||||
inputs_embeds: torch.Tensor | None = None,
|
|
||||||
**kwargs: object,
|
|
||||||
) -> torch.Tensor | IntermediateTensors:
|
|
||||||
if intermediate_tensors is not None:
|
|
||||||
inputs_embeds = None
|
|
||||||
|
|
||||||
hidden_states = self.language_model.model(
|
|
||||||
input_ids=input_ids,
|
|
||||||
positions=positions,
|
|
||||||
intermediate_tensors=intermediate_tensors,
|
|
||||||
inputs_embeds=inputs_embeds,
|
|
||||||
)
|
|
||||||
return hidden_states
|
|
||||||
|
|
||||||
def compute_logits(
|
|
||||||
self,
|
|
||||||
hidden_states: torch.Tensor,
|
|
||||||
) -> torch.Tensor | None:
|
|
||||||
return self.language_model.compute_logits(hidden_states)
|
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
||||||
loader = AutoWeightsLoader(self)
|
|
||||||
return loader.load_weights(weights)
|
|
||||||
@@ -123,7 +123,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
|||||||
qwen3_5_moe="Qwen3_5MoeConfig",
|
qwen3_5_moe="Qwen3_5MoeConfig",
|
||||||
laguna="LagunaConfig",
|
laguna="LagunaConfig",
|
||||||
lfm2_moe="Lfm2MoeConfig",
|
lfm2_moe="Lfm2MoeConfig",
|
||||||
tarsier2="Tarsier2Config",
|
|
||||||
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -86,7 +86,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
|||||||
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
|
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
|
||||||
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
|
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
|
||||||
"Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
|
"Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
|
||||||
"Tarsier2Config": "vllm.transformers_utils.configs.tarsier2",
|
|
||||||
# Special case: DeepseekV3Config is from HuggingFace Transformers
|
# Special case: DeepseekV3Config is from HuggingFace Transformers
|
||||||
"DeepseekV3Config": "transformers",
|
"DeepseekV3Config": "transformers",
|
||||||
}
|
}
|
||||||
@@ -161,7 +160,6 @@ __all__ = [
|
|||||||
"Qwen3_5TextConfig",
|
"Qwen3_5TextConfig",
|
||||||
"Qwen3_5MoeConfig",
|
"Qwen3_5MoeConfig",
|
||||||
"Qwen3_5MoeTextConfig",
|
"Qwen3_5MoeTextConfig",
|
||||||
"Tarsier2Config",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
from transformers import Qwen2VLConfig
|
|
||||||
|
|
||||||
|
|
||||||
class Tarsier2Config(Qwen2VLConfig):
|
|
||||||
"""
|
|
||||||
Tarsier2's config.json is written such that AutoConfig.from_pretrained will create
|
|
||||||
a deeply nested config consisting of:
|
|
||||||
|
|
||||||
- LlavaConfig
|
|
||||||
- Qwen2VLConfig
|
|
||||||
- Qwen2VLTextConfig
|
|
||||||
- Qwen2VLVisionConfig
|
|
||||||
- Qwen2VLConfig
|
|
||||||
- Qwen2VLTextConfig
|
|
||||||
- Qwen2VLVisionConfig
|
|
||||||
|
|
||||||
When it should really just be a single Qwen2VLConfig.
|
|
||||||
|
|
||||||
This class is a hack to stop AutoConfig from creating the nested config structure.
|
|
||||||
"""
|
|
||||||
|
|
||||||
model_type = "tarsier2"
|
|
||||||
Reference in New Issue
Block a user