[LoRA][Refactor] Unify multimodal LoRA token count hooks (#55071)

Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
linitra24
2026-09-14 13:19:15 +00:00
committed by GitHub
parent 6623fe5b4e
commit 9d4d9aa5bc
23 changed files with 276 additions and 295 deletions
+10 -6
View File
@@ -40,6 +40,7 @@ from vllm.multimodal.inputs import (
ImageItem,
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -1495,13 +1496,16 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
tower_model="visual.",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
merge_size = self.config.vision_config.spatial_merge_size
return num_image_tokens * merge_size**2
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
merge_size = self.config.vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
return num_mm_embeds * merge_size**2, num_mm_embeds
@MULTIMODAL_REGISTRY.register_processor(