[LoRA] Code cleanup (#60017)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
@@ -520,8 +521,12 @@ class BailingMoeV3VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
return num_image_tokens * self.visual.spatial_merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
return num_vision_tokens // self.visual.spatial_merge_size**2
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
return num_mm_embeds * self.visual.spatial_merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -82,6 +82,7 @@ from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -2085,24 +2086,16 @@ class CohereCompassForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
|
||||
return num_image_tokens * merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_vision_tokens // merge_size**2
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
merge_size = self.config.vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -31,7 +31,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
from typing_extensions import Self, TypeIs, deprecated
|
||||
from typing_extensions import Self, TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -422,17 +422,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
yield
|
||||
|
||||
@deprecated(
|
||||
"get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead."
|
||||
)
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ...
|
||||
|
||||
@deprecated(
|
||||
"get_num_mm_connector_tokens is deprecated; use "
|
||||
"get_mm_lora_token_counts instead."
|
||||
)
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ...
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
@@ -446,12 +435,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
del modality, mm_kwargs
|
||||
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
|
||||
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
|
||||
return (
|
||||
num_encoder_tokens,
|
||||
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} must implement get_mm_lora_token_counts "
|
||||
"to support tower/connector LoRA."
|
||||
)
|
||||
|
||||
def _embed_text_input_ids(
|
||||
|
||||
Reference in New Issue
Block a user