[LoRA][Refactor] Unify multimodal LoRA token count hooks (#55071)
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
@@ -54,6 +54,7 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -718,13 +719,16 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
|
||||
return image_embeds.split(sizes)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
merge_size = self.vision_tower.spatial_merge_size
|
||||
return num_image_tokens * (merge_size**2)
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
merge_size = self.vision_tower.spatial_merge_size
|
||||
return num_vision_tokens // (merge_size**2)
|
||||
return num_mm_embeds * (merge_size**2), num_mm_embeds
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
Reference in New Issue
Block a user