[New Model] support new model ovis2.6 (#34426)
Signed-off-by: myselvess <23743269+myselvess@users.noreply.github.com>
This commit is contained in:
@@ -42,21 +42,12 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
|
||||
IMAGE_TOKEN = "<image>"
|
||||
IMAGE_PLACEHOLDER_ID = 151669
|
||||
VIDEO_TOKEN = "<video>"
|
||||
INDICATOR_IDS = [-301, -302, -303, -304]
|
||||
|
||||
IMAGE_PAD_TOKEN_MAP = {
|
||||
"gemma2": "<unused0>",
|
||||
"llama": "<|reserved_special_token_0|>",
|
||||
"qwen2": "<|image_pad|>",
|
||||
"qwen3": "<|image_pad|>",
|
||||
}
|
||||
IMAGE_PAD_TOKEN_ID_MAP = {
|
||||
"gemma2": 7,
|
||||
"llama": 128002,
|
||||
"qwen2": 151655,
|
||||
"qwen3": 151655,
|
||||
}
|
||||
VIDEO_PLACEHOLDER_ID = 151670
|
||||
INDICATOR_IDS = [151672, 151673, 151674, 151675]
|
||||
IMAGE_PAD_TOKEN_ID = 151655
|
||||
THINK_END_TOKEN_ID = 151668
|
||||
|
||||
|
||||
class Ovis2_5ImagePatchInputs(TensorSchema):
|
||||
@@ -187,17 +178,11 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
vit_config = self.get_hf_config().vit_config
|
||||
return self.ctx.get_hf_processor(
|
||||
Ovis2_5Processor,
|
||||
image_pad_token=self.get_image_pad_token(),
|
||||
patch_size=vit_config.patch_size,
|
||||
hidden_stride=vit_config.hidden_stride,
|
||||
temporal_patch_size=vit_config.temporal_patch_size,
|
||||
)
|
||||
|
||||
def get_image_pad_token(self) -> str:
|
||||
hf_text_config = self.get_hf_config().get_text_config()
|
||||
text_model_type = hf_text_config.model_type
|
||||
return IMAGE_PAD_TOKEN_MAP.get(text_model_type)
|
||||
|
||||
def get_image_processor(self) -> BaseImageProcessor:
|
||||
return self.get_hf_processor().image_processor # type: ignore
|
||||
|
||||
@@ -342,9 +327,9 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
hf_config = self.info.get_hf_config()
|
||||
vte_vocab_size = hf_config.visual_vocab_size
|
||||
return [
|
||||
vte_vocab_size - len(INDICATOR_IDS) + abs(x + 300) - 1
|
||||
vte_vocab_size - len(INDICATOR_IDS) + (x - INDICATOR_IDS[0])
|
||||
for x in visual_indicators
|
||||
if x < -300
|
||||
if x >= INDICATOR_IDS[0]
|
||||
]
|
||||
|
||||
def _call_hf_processor(
|
||||
@@ -417,6 +402,14 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> list[PromptReplacement]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
placeholder = {
|
||||
"image": vocab[IMAGE_TOKEN],
|
||||
"video": vocab[VIDEO_TOKEN],
|
||||
}
|
||||
|
||||
def get_replacement_ovis(item_idx, modality: str):
|
||||
if modality == "image":
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
@@ -432,7 +425,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=IMAGE_TOKEN if modality == "image" else VIDEO_TOKEN,
|
||||
target=[placeholder[modality]],
|
||||
replacement=partial(get_replacement_ovis, modality=modality),
|
||||
)
|
||||
for modality in ("image", "video")
|
||||
@@ -476,8 +469,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
)
|
||||
self.vte = VisualEmbedding(config.visual_vocab_size, config.hidden_size)
|
||||
|
||||
text_model_type = self.config.get_text_config().model_type
|
||||
self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
|
||||
self.image_pad_token_id: int = IMAGE_PAD_TOKEN_ID
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.get_language_model().make_empty_intermediate_tensors
|
||||
|
||||
@@ -425,6 +425,8 @@ _MULTIMODAL_MODELS = {
|
||||
),
|
||||
"Ovis": ("ovis", "Ovis"),
|
||||
"Ovis2_5": ("ovis2_5", "Ovis2_5"),
|
||||
"Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
|
||||
"Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
|
||||
"PaddleOCRVLForConditionalGeneration": (
|
||||
"paddleocr_vl",
|
||||
"PaddleOCRVLForConditionalGeneration",
|
||||
|
||||
@@ -582,7 +582,6 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
hidden_states = self.embeddings(pixel_values, grid_thws)
|
||||
|
||||
last_hidden_state = self.encoder(hidden_states, grid_thws)
|
||||
last_hidden_state = self.post_layernorm(last_hidden_state)
|
||||
|
||||
return last_hidden_state
|
||||
|
||||
|
||||
Reference in New Issue
Block a user