[LoRA][Gemma4] Support vision tower LoRA (#42662)

Signed-off-by: bk-201 <joy25810@foxmail.com>
Signed-off-by: linitra24 <Joy25810@foxmail.com>
Signed-off-by: linitra24 <joy25810@gmail.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: linitra24 <joy25810@gmail.com>
This commit is contained in:
linitra24
2026-08-13 22:07:43 +08:00
committed by GitHub
co-authored by Jee Jee Li linitra24
parent 8e1131e1d1
commit 95c9144424
10 changed files with 309 additions and 48 deletions
+120 -14
View File
@@ -50,6 +50,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -1037,6 +1038,10 @@ class Gemma4ForConditionalGeneration(
self.multimodal_config = multimodal_config
self.model_dtype = vllm_config.model_config.dtype
self.vllm_config = vllm_config
lora_config = vllm_config.lora_config
self._enable_mm_lora = bool(
lora_config is not None and lora_config.enable_tower_connector_lora
)
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
@@ -1265,8 +1270,8 @@ class Gemma4ForConditionalGeneration(
Groups images by patch count (resolution bucket) so each
encoder call processes a uniform-shape batch with no
cross-resolution padding. Pooling and projection are then
applied over a single concatenated tensor for all images.
cross-resolution padding. With MM LoRA enabled, all images are
padded into one batch so the encoder call matches the tower mapping.
"""
pixel_values = image_input["pixel_values"]
pixel_position_ids = image_input["pixel_position_ids"]
@@ -1284,24 +1289,65 @@ class Gemma4ForConditionalGeneration(
if isinstance(pixel_values, list)
else pixel_values.shape[0]
)
pool_position_ids = pixel_position_ids
for idx in range(total_images):
pv = pixel_values[idx]
pp = pixel_position_ids[idx]
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
if self._enable_mm_lora:
max_soft_tokens = vision_cfg.default_output_length
mm_processor_kwargs = getattr(
getattr(self, "multimodal_config", None),
"mm_processor_kwargs",
None,
)
if isinstance(mm_processor_kwargs, Mapping):
value, _ = _get_max_soft_tokens(mm_processor_kwargs)
if isinstance(value, int) and value in _SUPPORTED_SOFT_TOKENS:
max_soft_tokens = value
max_patches = max_soft_tokens * pooling_k2
padded_position_ids: list[torch.Tensor] = []
for idx in range(total_images):
pv = pixel_values[idx]
pp = pixel_position_ids[idx]
num_patches = pv.shape[0]
if num_patches > max_patches:
raise ValueError(
f"Image {idx} has {num_patches} patches, which exceeds "
f"the MM LoRA patch limit of {max_patches}."
)
pad_len = max_patches - num_patches
pv = torch.cat(
(pv, pv.new_zeros((pad_len, *pv.shape[1:]))),
dim=0,
)
pp = torch.cat(
(pp, pp.new_full((pad_len, *pp.shape[1:]), -1)),
dim=0,
)
buckets.setdefault(max_patches, []).append((idx, pv, pp))
padded_position_ids.append(pp)
pool_position_ids = padded_position_ids
else:
for idx in range(total_images):
pv = pixel_values[idx]
pp = pixel_position_ids[idx]
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
# Encode each resolution bucket in memory-safe chunks. Re-read
# free memory per bucket because the previous bucket's encoder
# pass has already allocated activations we should account for.
last_hidden_states_map: dict[int, torch.Tensor] = {}
for patches, items in buckets.items():
free, total = torch.accelerator.get_memory_info()
max_batch_size = min(
len(items),
self._encoder_chunk(
patches, free, total, vision_cfg.position_embedding_size
),
)
if self._enable_mm_lora:
max_batch_size = len(items)
else:
free, total = torch.accelerator.get_memory_info()
max_batch_size = min(
len(items),
self._encoder_chunk(
patches, free, total, vision_cfg.position_embedding_size
),
)
for chunk_idx in range(0, len(items), max_batch_size):
chunk_items = items[chunk_idx : chunk_idx + max_batch_size]
@@ -1338,7 +1384,7 @@ class Gemma4ForConditionalGeneration(
output_length = chunk_hidden.shape[0] // pooling_k2
single_hidden = chunk_hidden.unsqueeze(0)
single_pos_ids = pixel_position_ids[orig_idx].unsqueeze(0)
single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
padding_positions = (single_pos_ids == -1).all(dim=-1)
pooled_states, valid_mask = vt.pooler(
@@ -2146,6 +2192,66 @@ class Gemma4ForConditionalGeneration(
tower_model=tower_models,
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
if modality in ("image", "video"):
vision_config = self.config.vision_config
pooling_k2 = vision_config.pooling_kernel_size**2
if modality == "image":
pixel_values_key = "pixel_values"
max_soft_tokens = vision_config.default_output_length
mm_processor_kwargs = getattr(
getattr(self, "multimodal_config", None),
"mm_processor_kwargs",
None,
)
if isinstance(mm_processor_kwargs, Mapping):
val, _ = _get_max_soft_tokens(mm_processor_kwargs)
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
max_soft_tokens = val
else:
pixel_values_key = "pixel_values_videos"
max_soft_tokens = _VIDEO_MAX_SOFT_TOKENS
tower_tokens = max_soft_tokens * pooling_k2 if modality == "image" else None
connector_tokens = num_mm_embeds
if tower_tokens is None and mm_kwargs is not None:
field = mm_kwargs.get(pixel_values_key)
if field is not None:
data = field.data
if isinstance(data, torch.Tensor) and data.ndim >= 2:
tower_tokens = int(math.prod(data.shape[:-1]))
if tower_tokens is None:
min_soft_tokens = min(_SUPPORTED_SOFT_TOKENS)
tower_tokens = (
math.ceil(num_mm_embeds / min_soft_tokens)
* max_soft_tokens
* pooling_k2
)
if modality == "audio":
tower_tokens = num_mm_embeds
connector_tokens = num_mm_embeds
if mm_kwargs is not None:
field = mm_kwargs.get("input_features_padded")
if field is not None:
data = field.data
if isinstance(data, torch.Tensor) and data.ndim >= 2:
batch_size = math.prod(data.shape[:-2])
audio_tokens = batch_size * math.ceil(data.shape[-2] / 4)
tower_tokens = audio_tokens
connector_tokens = audio_tokens
return tower_tokens, connector_tokens
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality == "image":
+23 -1
View File
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""
...
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: "MultiModalKwargsItem | None",
num_mm_embeds: int,
) -> tuple[int, int | None]:
"""
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
MM LoRA uses these counts to build adapter mappings for the tower and
connector forwards. Models with multiple modalities can override this
when each modality has different encoder padding or pooling behavior.
"""
del modality, mm_kwargs
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
return (
num_encoder_tokens,
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
)
@overload
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...