[LoRA][Refactor] Unify multimodal LoRA token count hooks (#55071)

Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
linitra24
2026-09-14 13:19:15 +00:00
committed by GitHub
parent 6623fe5b4e
commit 9d4d9aa5bc
23 changed files with 276 additions and 295 deletions
+9 -14
View File
@@ -28,6 +28,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import BaseMultiModalProcessorCache
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -739,18 +740,12 @@ class LlavaForConditionalGeneration(
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
# LLaVA's vision encoder outputs one token per patch without
# spatial merging or pixel shuffle
return num_image_tokens
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
# LLaVA's MLP projector outputs the same number of tokens
# as it receives from the vision encoder (1:1 mapping)
return num_vision_tokens
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
return num_mm_embeds, num_mm_embeds