[LoRA][Gemma4] Support vision tower LoRA (#42662)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: linitra24 <Joy25810@foxmail.com> Signed-off-by: linitra24 <joy25810@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: linitra24 <joy25810@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
linitra24
parent
8e1131e1d1
commit
95c9144424
@@ -50,6 +50,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -1037,6 +1038,10 @@ class Gemma4ForConditionalGeneration(
|
||||
self.multimodal_config = multimodal_config
|
||||
self.model_dtype = vllm_config.model_config.dtype
|
||||
self.vllm_config = vllm_config
|
||||
lora_config = vllm_config.lora_config
|
||||
self._enable_mm_lora = bool(
|
||||
lora_config is not None and lora_config.enable_tower_connector_lora
|
||||
)
|
||||
|
||||
# Only quantize towers when the quant method supports their
|
||||
# dimensions. BNB/torchao handle arbitrary sizes; other methods
|
||||
@@ -1265,8 +1270,8 @@ class Gemma4ForConditionalGeneration(
|
||||
|
||||
Groups images by patch count (resolution bucket) so each
|
||||
encoder call processes a uniform-shape batch with no
|
||||
cross-resolution padding. Pooling and projection are then
|
||||
applied over a single concatenated tensor for all images.
|
||||
cross-resolution padding. With MM LoRA enabled, all images are
|
||||
padded into one batch so the encoder call matches the tower mapping.
|
||||
"""
|
||||
pixel_values = image_input["pixel_values"]
|
||||
pixel_position_ids = image_input["pixel_position_ids"]
|
||||
@@ -1284,24 +1289,65 @@ class Gemma4ForConditionalGeneration(
|
||||
if isinstance(pixel_values, list)
|
||||
else pixel_values.shape[0]
|
||||
)
|
||||
pool_position_ids = pixel_position_ids
|
||||
|
||||
for idx in range(total_images):
|
||||
pv = pixel_values[idx]
|
||||
pp = pixel_position_ids[idx]
|
||||
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
|
||||
if self._enable_mm_lora:
|
||||
max_soft_tokens = vision_cfg.default_output_length
|
||||
mm_processor_kwargs = getattr(
|
||||
getattr(self, "multimodal_config", None),
|
||||
"mm_processor_kwargs",
|
||||
None,
|
||||
)
|
||||
if isinstance(mm_processor_kwargs, Mapping):
|
||||
value, _ = _get_max_soft_tokens(mm_processor_kwargs)
|
||||
if isinstance(value, int) and value in _SUPPORTED_SOFT_TOKENS:
|
||||
max_soft_tokens = value
|
||||
|
||||
max_patches = max_soft_tokens * pooling_k2
|
||||
padded_position_ids: list[torch.Tensor] = []
|
||||
for idx in range(total_images):
|
||||
pv = pixel_values[idx]
|
||||
pp = pixel_position_ids[idx]
|
||||
num_patches = pv.shape[0]
|
||||
if num_patches > max_patches:
|
||||
raise ValueError(
|
||||
f"Image {idx} has {num_patches} patches, which exceeds "
|
||||
f"the MM LoRA patch limit of {max_patches}."
|
||||
)
|
||||
|
||||
pad_len = max_patches - num_patches
|
||||
pv = torch.cat(
|
||||
(pv, pv.new_zeros((pad_len, *pv.shape[1:]))),
|
||||
dim=0,
|
||||
)
|
||||
pp = torch.cat(
|
||||
(pp, pp.new_full((pad_len, *pp.shape[1:]), -1)),
|
||||
dim=0,
|
||||
)
|
||||
buckets.setdefault(max_patches, []).append((idx, pv, pp))
|
||||
padded_position_ids.append(pp)
|
||||
pool_position_ids = padded_position_ids
|
||||
else:
|
||||
for idx in range(total_images):
|
||||
pv = pixel_values[idx]
|
||||
pp = pixel_position_ids[idx]
|
||||
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
|
||||
|
||||
# Encode each resolution bucket in memory-safe chunks. Re-read
|
||||
# free memory per bucket because the previous bucket's encoder
|
||||
# pass has already allocated activations we should account for.
|
||||
last_hidden_states_map: dict[int, torch.Tensor] = {}
|
||||
for patches, items in buckets.items():
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
max_batch_size = min(
|
||||
len(items),
|
||||
self._encoder_chunk(
|
||||
patches, free, total, vision_cfg.position_embedding_size
|
||||
),
|
||||
)
|
||||
if self._enable_mm_lora:
|
||||
max_batch_size = len(items)
|
||||
else:
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
max_batch_size = min(
|
||||
len(items),
|
||||
self._encoder_chunk(
|
||||
patches, free, total, vision_cfg.position_embedding_size
|
||||
),
|
||||
)
|
||||
|
||||
for chunk_idx in range(0, len(items), max_batch_size):
|
||||
chunk_items = items[chunk_idx : chunk_idx + max_batch_size]
|
||||
@@ -1338,7 +1384,7 @@ class Gemma4ForConditionalGeneration(
|
||||
output_length = chunk_hidden.shape[0] // pooling_k2
|
||||
|
||||
single_hidden = chunk_hidden.unsqueeze(0)
|
||||
single_pos_ids = pixel_position_ids[orig_idx].unsqueeze(0)
|
||||
single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
|
||||
padding_positions = (single_pos_ids == -1).all(dim=-1)
|
||||
|
||||
pooled_states, valid_mask = vt.pooler(
|
||||
@@ -2146,6 +2192,66 @@ class Gemma4ForConditionalGeneration(
|
||||
tower_model=tower_models,
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
if modality in ("image", "video"):
|
||||
vision_config = self.config.vision_config
|
||||
pooling_k2 = vision_config.pooling_kernel_size**2
|
||||
|
||||
if modality == "image":
|
||||
pixel_values_key = "pixel_values"
|
||||
max_soft_tokens = vision_config.default_output_length
|
||||
mm_processor_kwargs = getattr(
|
||||
getattr(self, "multimodal_config", None),
|
||||
"mm_processor_kwargs",
|
||||
None,
|
||||
)
|
||||
if isinstance(mm_processor_kwargs, Mapping):
|
||||
val, _ = _get_max_soft_tokens(mm_processor_kwargs)
|
||||
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
|
||||
max_soft_tokens = val
|
||||
else:
|
||||
pixel_values_key = "pixel_values_videos"
|
||||
max_soft_tokens = _VIDEO_MAX_SOFT_TOKENS
|
||||
|
||||
tower_tokens = max_soft_tokens * pooling_k2 if modality == "image" else None
|
||||
connector_tokens = num_mm_embeds
|
||||
if tower_tokens is None and mm_kwargs is not None:
|
||||
field = mm_kwargs.get(pixel_values_key)
|
||||
if field is not None:
|
||||
data = field.data
|
||||
if isinstance(data, torch.Tensor) and data.ndim >= 2:
|
||||
tower_tokens = int(math.prod(data.shape[:-1]))
|
||||
|
||||
if tower_tokens is None:
|
||||
min_soft_tokens = min(_SUPPORTED_SOFT_TOKENS)
|
||||
tower_tokens = (
|
||||
math.ceil(num_mm_embeds / min_soft_tokens)
|
||||
* max_soft_tokens
|
||||
* pooling_k2
|
||||
)
|
||||
|
||||
if modality == "audio":
|
||||
tower_tokens = num_mm_embeds
|
||||
connector_tokens = num_mm_embeds
|
||||
|
||||
if mm_kwargs is not None:
|
||||
field = mm_kwargs.get("input_features_padded")
|
||||
if field is not None:
|
||||
data = field.data
|
||||
if isinstance(data, torch.Tensor) and data.ndim >= 2:
|
||||
batch_size = math.prod(data.shape[:-2])
|
||||
audio_tokens = batch_size * math.ceil(data.shape[-2] / 4)
|
||||
tower_tokens = audio_tokens
|
||||
connector_tokens = audio_tokens
|
||||
|
||||
return tower_tokens, connector_tokens
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality == "image":
|
||||
|
||||
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: "MultiModalKwargsItem | None",
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
"""
|
||||
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
|
||||
MM LoRA uses these counts to build adapter mappings for the tower and
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
del modality, mm_kwargs
|
||||
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
|
||||
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
|
||||
return (
|
||||
num_encoder_tokens,
|
||||
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
||||
)
|
||||
|
||||
@overload
|
||||
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user