diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml
index a79196ffbd..4287816b90 100644
--- a/.buildkite/test_areas/lora.yaml
+++ b/.buildkite/test_areas/lora.yaml
@@ -48,3 +48,4 @@ steps:
- pytest -v -s -x lora/test_olmoe_tp.py
- pytest -v -s -x lora/test_gptoss_tp.py
- pytest -v -s -x lora/test_qwen35_densemodel_lora.py
+ - pytest -v -s -x lora/test_gemma4_tp.py
diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 64c7c7062a..d7b587b6df 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -523,7 +523,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + IE+ | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
| `Gemma3ForConditionalGeneration` | Gemma 3 | T + IE+ | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ |
| `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
-| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I+ + V + A* | `google/gemma-4-E2B-it`, etc. | | ✅︎ |
+| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I+ + V + A* | `google/gemma-4-E2B-it`, etc. | ✅︎ | ✅︎ |
| `Gemma4UnifiedForConditionalGeneration` | Gemma 4 Unified | T + I+ + V + A | `google/gemma-4-12B-it`, etc. | | ✅︎ |
| `GLM4VForCausalLM`^ | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ |
| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + IE+ + VE+ | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ |
diff --git a/tests/lora/conftest.py b/tests/lora/conftest.py
index 899b3129d6..f14b488a17 100644
--- a/tests/lora/conftest.py
+++ b/tests/lora/conftest.py
@@ -253,6 +253,11 @@ def qwen3vl_vision_lora_files():
)
+@pytest.fixture(scope="session")
+def gemma4_vision_lora_files():
+ return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon")
+
+
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 Meow LoRA files once per test session."""
diff --git a/tests/lora/test_gemma4_tp.py b/tests/lora/test_gemma4_tp.py
new file mode 100644
index 0000000000..efbee00d62
--- /dev/null
+++ b/tests/lora/test_gemma4_tp.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# NOTE To avoid overloading the CI pipeline, this test script will not
+# be triggered on CI and is primarily intended for local testing and verification.
+
+import vllm
+from vllm.assets.image import ImageAsset
+from vllm.lora.request import LoRARequest
+
+from ..utils import multi_gpu_test
+
+MODEL_PATH = "google/gemma-4-E2B-it"
+
+PROMPT_TEMPLATE = """<|turn>user
+<|image|>What is in the image?
+<|turn>model
+"""
+
+TEST_IMAGES = [
+ ImageAsset("stop_sign"),
+ ImageAsset("cherry_blossom"),
+]
+
+EXPECTED_OUTPUTS_VISION = [
+ "A red stop sign stands prominently in the foreground.",
+ "A majestic skyscraper stands tall, partially obscured by a vibrant "
+ "canopy of cherry blossoms, against a clear blue sky.",
+]
+
+
+def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
+ prompts = [
+ {
+ "prompt": PROMPT_TEMPLATE,
+ "multi_modal_data": {"image": asset.pil_image},
+ }
+ for asset in TEST_IMAGES
+ ]
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=128)
+ outputs = llm.generate(
+ prompts,
+ sampling_params,
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
+ )
+
+ generated_texts = [output.outputs[0].text.strip() for output in outputs]
+ for generated, expected in zip(generated_texts, EXPECTED_OUTPUTS_VISION):
+ assert generated.startswith(expected), (
+ f"Generated text {generated!r} does not match expected {expected!r}"
+ )
+
+
+def test_gemma4_lora(gemma4_vision_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ enforce_eager=True,
+ max_loras=4,
+ trust_remote_code=True,
+ limit_mm_per_prompt={"image": 1},
+ mm_processor_cache_gb=0,
+ enable_tower_connector_lora=True,
+ )
+
+ generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
+ generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
+
+
+@multi_gpu_test(num_gpus=2)
+def test_gemma4_lora_tp2(gemma4_vision_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enforce_eager=True,
+ enable_lora=True,
+ max_loras=4,
+ trust_remote_code=True,
+ tensor_parallel_size=2,
+ limit_mm_per_prompt={"image": 1},
+ mm_processor_cache_gb=0,
+ enable_tower_connector_lora=True,
+ )
+
+ generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
+ generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
+
+
+@multi_gpu_test(num_gpus=4)
+def test_gemma4_lora_tp4(gemma4_vision_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enforce_eager=True,
+ enable_lora=True,
+ max_loras=4,
+ trust_remote_code=True,
+ tensor_parallel_size=4,
+ limit_mm_per_prompt={"image": 1},
+ mm_processor_cache_gb=0,
+ enable_tower_connector_lora=True,
+ )
+
+ generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
+ generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py
index 2db024dc6b..06e3fe4fa7 100644
--- a/tests/v1/worker/test_gpu_model_runner.py
+++ b/tests/v1/worker/test_gpu_model_runner.py
@@ -380,9 +380,13 @@ def test_select_common_block_size_no_valid_option():
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
model = Mock()
- model.get_num_mm_encoder_tokens.side_effect = lambda num_embeds: num_embeds + 1
+ model.get_mm_lora_token_counts.side_effect = (
+ lambda *, modality, mm_kwargs, num_mm_embeds: (
+ num_mm_embeds + 1,
+ num_mm_embeds + 11,
+ )
+ )
model.get_mm_mapping.return_value = SimpleNamespace(connector=True)
- model.get_num_mm_connector_tokens.side_effect = lambda num_tokens: num_tokens + 10
lora_manager = Mock()
lora_manager.supports_tower_connector_lora.return_value = True
diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py
index bd5e91cfa5..5db3cd4faa 100644
--- a/vllm/lora/model_manager.py
+++ b/vllm/lora/model_manager.py
@@ -247,8 +247,17 @@ class LoRAModelManager:
mm_budget = MultiModalBudget(vllm_config, mm_registry)
limit_per_prompt = max(mm_budget.mm_max_items_per_prompt.values())
- num_encoder_tokens = self.model.get_num_mm_encoder_tokens(
- mm_budget.get_encoder_budget()
+ max_lora_tokens = mm_budget.get_encoder_budget()
+ lora_token_counts_by_modality = [
+ self.model.get_mm_lora_token_counts(
+ modality=modality,
+ mm_kwargs=None,
+ num_mm_embeds=max_lora_tokens,
+ )
+ for modality in mm_budget.mm_max_toks_per_item
+ ]
+ num_encoder_tokens = max(
+ tower_tokens for tower_tokens, _ in lora_token_counts_by_modality
)
# Tower wrappers
@@ -263,10 +272,15 @@ class LoRAModelManager:
# Use wrapper for connector if present.
if self.mm_mapping.connector:
- if hasattr(self.model, "get_num_mm_connector_tokens"):
- connector_tokens = self.model.get_num_mm_connector_tokens(
- num_encoder_tokens
- )
+ connector_tokens = max(
+ (
+ connector_tokens
+ for _, connector_tokens in lora_token_counts_by_modality
+ if connector_tokens is not None
+ ),
+ default=None,
+ )
+ if connector_tokens is not None:
connector_punica_wrapper = get_punica_wrapper(
connector_tokens,
max_batches=self.max_num_seqs * limit_per_prompt,
diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py
index 1ef236907f..1bfc0c0b7f 100644
--- a/vllm/model_executor/models/gemma4_mm.py
+++ b/vllm/model_executor/models/gemma4_mm.py
@@ -50,6 +50,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
+ MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -1037,6 +1038,10 @@ class Gemma4ForConditionalGeneration(
self.multimodal_config = multimodal_config
self.model_dtype = vllm_config.model_config.dtype
self.vllm_config = vllm_config
+ lora_config = vllm_config.lora_config
+ self._enable_mm_lora = bool(
+ lora_config is not None and lora_config.enable_tower_connector_lora
+ )
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
@@ -1265,8 +1270,8 @@ class Gemma4ForConditionalGeneration(
Groups images by patch count (resolution bucket) so each
encoder call processes a uniform-shape batch with no
- cross-resolution padding. Pooling and projection are then
- applied over a single concatenated tensor for all images.
+ cross-resolution padding. With MM LoRA enabled, all images are
+ padded into one batch so the encoder call matches the tower mapping.
"""
pixel_values = image_input["pixel_values"]
pixel_position_ids = image_input["pixel_position_ids"]
@@ -1284,24 +1289,65 @@ class Gemma4ForConditionalGeneration(
if isinstance(pixel_values, list)
else pixel_values.shape[0]
)
+ pool_position_ids = pixel_position_ids
- for idx in range(total_images):
- pv = pixel_values[idx]
- pp = pixel_position_ids[idx]
- buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
+ if self._enable_mm_lora:
+ max_soft_tokens = vision_cfg.default_output_length
+ mm_processor_kwargs = getattr(
+ getattr(self, "multimodal_config", None),
+ "mm_processor_kwargs",
+ None,
+ )
+ if isinstance(mm_processor_kwargs, Mapping):
+ value, _ = _get_max_soft_tokens(mm_processor_kwargs)
+ if isinstance(value, int) and value in _SUPPORTED_SOFT_TOKENS:
+ max_soft_tokens = value
+
+ max_patches = max_soft_tokens * pooling_k2
+ padded_position_ids: list[torch.Tensor] = []
+ for idx in range(total_images):
+ pv = pixel_values[idx]
+ pp = pixel_position_ids[idx]
+ num_patches = pv.shape[0]
+ if num_patches > max_patches:
+ raise ValueError(
+ f"Image {idx} has {num_patches} patches, which exceeds "
+ f"the MM LoRA patch limit of {max_patches}."
+ )
+
+ pad_len = max_patches - num_patches
+ pv = torch.cat(
+ (pv, pv.new_zeros((pad_len, *pv.shape[1:]))),
+ dim=0,
+ )
+ pp = torch.cat(
+ (pp, pp.new_full((pad_len, *pp.shape[1:]), -1)),
+ dim=0,
+ )
+ buckets.setdefault(max_patches, []).append((idx, pv, pp))
+ padded_position_ids.append(pp)
+ pool_position_ids = padded_position_ids
+ else:
+ for idx in range(total_images):
+ pv = pixel_values[idx]
+ pp = pixel_position_ids[idx]
+ buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
# Encode each resolution bucket in memory-safe chunks. Re-read
# free memory per bucket because the previous bucket's encoder
# pass has already allocated activations we should account for.
last_hidden_states_map: dict[int, torch.Tensor] = {}
for patches, items in buckets.items():
- free, total = torch.accelerator.get_memory_info()
- max_batch_size = min(
- len(items),
- self._encoder_chunk(
- patches, free, total, vision_cfg.position_embedding_size
- ),
- )
+ if self._enable_mm_lora:
+ max_batch_size = len(items)
+ else:
+ free, total = torch.accelerator.get_memory_info()
+ max_batch_size = min(
+ len(items),
+ self._encoder_chunk(
+ patches, free, total, vision_cfg.position_embedding_size
+ ),
+ )
for chunk_idx in range(0, len(items), max_batch_size):
chunk_items = items[chunk_idx : chunk_idx + max_batch_size]
@@ -1338,7 +1384,7 @@ class Gemma4ForConditionalGeneration(
output_length = chunk_hidden.shape[0] // pooling_k2
single_hidden = chunk_hidden.unsqueeze(0)
- single_pos_ids = pixel_position_ids[orig_idx].unsqueeze(0)
+ single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
padding_positions = (single_pos_ids == -1).all(dim=-1)
pooled_states, valid_mask = vt.pooler(
@@ -2146,6 +2192,66 @@ class Gemma4ForConditionalGeneration(
tower_model=tower_models,
)
+ def get_mm_lora_token_counts(
+ self,
+ *,
+ modality: str,
+ mm_kwargs: MultiModalKwargsItem | None,
+ num_mm_embeds: int,
+ ) -> tuple[int, int | None]:
+ if modality in ("image", "video"):
+ vision_config = self.config.vision_config
+ pooling_k2 = vision_config.pooling_kernel_size**2
+
+ if modality == "image":
+ pixel_values_key = "pixel_values"
+ max_soft_tokens = vision_config.default_output_length
+ mm_processor_kwargs = getattr(
+ getattr(self, "multimodal_config", None),
+ "mm_processor_kwargs",
+ None,
+ )
+ if isinstance(mm_processor_kwargs, Mapping):
+ val, _ = _get_max_soft_tokens(mm_processor_kwargs)
+ if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
+ max_soft_tokens = val
+ else:
+ pixel_values_key = "pixel_values_videos"
+ max_soft_tokens = _VIDEO_MAX_SOFT_TOKENS
+
+ tower_tokens = max_soft_tokens * pooling_k2 if modality == "image" else None
+ connector_tokens = num_mm_embeds
+ if tower_tokens is None and mm_kwargs is not None:
+ field = mm_kwargs.get(pixel_values_key)
+ if field is not None:
+ data = field.data
+ if isinstance(data, torch.Tensor) and data.ndim >= 2:
+ tower_tokens = int(math.prod(data.shape[:-1]))
+
+ if tower_tokens is None:
+ min_soft_tokens = min(_SUPPORTED_SOFT_TOKENS)
+ tower_tokens = (
+ math.ceil(num_mm_embeds / min_soft_tokens)
+ * max_soft_tokens
+ * pooling_k2
+ )
+
+ if modality == "audio":
+ tower_tokens = num_mm_embeds
+ connector_tokens = num_mm_embeds
+
+ if mm_kwargs is not None:
+ field = mm_kwargs.get("input_features_padded")
+ if field is not None:
+ data = field.data
+ if isinstance(data, torch.Tensor) and data.ndim >= 2:
+ batch_size = math.prod(data.shape[:-2])
+ audio_tokens = batch_size * math.ceil(data.shape[-2] / 4)
+ tower_tokens = audio_tokens
+ connector_tokens = audio_tokens
+
+ return tower_tokens, connector_tokens
+
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality == "image":
diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py
index 46d0ed30df..38fca287d5 100644
--- a/vllm/model_executor/models/interfaces.py
+++ b/vllm/model_executor/models/interfaces.py
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.utils import WeightsMapper
- from vllm.multimodal.inputs import MultiModalFeatureSpec
+ from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""
...
+ def get_mm_lora_token_counts(
+ self,
+ *,
+ modality: str,
+ mm_kwargs: "MultiModalKwargsItem | None",
+ num_mm_embeds: int,
+ ) -> tuple[int, int | None]:
+ """
+ Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
+
+ MM LoRA uses these counts to build adapter mappings for the tower and
+ connector forwards. Models with multiple modalities can override this
+ when each modality has different encoder padding or pooling behavior.
+ """
+ del modality, mm_kwargs
+ num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
+ num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
+ return (
+ num_encoder_tokens,
+ num_connector_tokens if isinstance(num_connector_tokens, int) else None,
+ )
+
@overload
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
diff --git a/vllm/v1/worker/gpu/mm/lora.py b/vllm/v1/worker/gpu/mm/lora.py
index 492914a74b..819f3b1cb3 100644
--- a/vllm/v1/worker/gpu/mm/lora.py
+++ b/vllm/v1/worker/gpu/mm/lora.py
@@ -29,6 +29,7 @@ def set_active_mm_loras(
token_lora_mapping: list[int] = []
lora_requests = set()
encoder_token_counts: list[int] = []
+ connector_token_counts: list[int] = []
# iterate through images
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
@@ -42,10 +43,17 @@ def set_active_mm_loras(
# iterate through visual tokens
for mm_input_id in encoder_input_ids:
pos_info = mm_features[mm_input_id].mm_position
- num_tokens = model.get_num_mm_encoder_tokens(pos_info.get_num_embeds())
+
+ tower_tokens, connector_tokens = model.get_mm_lora_token_counts(
+ modality=mm_features[mm_input_id].modality,
+ mm_kwargs=mm_features[mm_input_id].data,
+ num_mm_embeds=pos_info.get_num_embeds(),
+ )
+
prompt_lora_mapping.append(lora_id)
- token_lora_mapping.extend([lora_id] * num_tokens)
- encoder_token_counts.append(num_tokens)
+ token_lora_mapping.extend([lora_id] * tower_tokens)
+ encoder_token_counts.append(tower_tokens)
+ connector_token_counts.append(connector_tokens)
if lora_id > 0:
lora_request = lora_state.lora_requests.get(req_id)
@@ -69,19 +77,13 @@ def set_active_mm_loras(
if (
mm_mapping is None
or not mm_mapping.connector
- or not hasattr(model, "get_num_mm_connector_tokens")
+ or not all(count is not None for count in connector_token_counts)
):
return
connector_token_mapping = np.repeat(
np.array(prompt_lora_mapping, dtype=np.int32),
- np.array(
- [
- model.get_num_mm_connector_tokens(num_tokens)
- for num_tokens in encoder_token_counts
- ],
- dtype=np.int32,
- ),
+ np.array(connector_token_counts, dtype=np.int32),
)
lora_manager.set_active_adapters(
lora_requests,
diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index defaccacf6..e81913f928 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -3123,18 +3123,24 @@ class GPUModelRunner(
token_lora_mapping = []
lora_requests = set()
encoder_token_counts = []
+ connector_token_counts = []
- for req_id, pos_info in mm_lora_refs:
+ for (req_id, pos_info), (modality, mm_item) in zip(
+ mm_lora_refs,
+ mm_kwargs,
+ ):
req_idx = self.input_batch.req_id_to_index[req_id]
lora_id = int(self.input_batch.request_lora_mapping[req_idx])
- # Prefer pos_info.get_num_embeds to count precise MM embedding tokens.
- num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]
- pos_info.get_num_embeds()
+ tower_tokens, connector_tokens = self.model.get_mm_lora_token_counts( # type: ignore[attr-defined]
+ modality=modality,
+ mm_kwargs=mm_item,
+ num_mm_embeds=pos_info.get_num_embeds(),
)
prompt_lora_mapping.append(lora_id)
- token_lora_mapping.extend([lora_id] * num_tokens)
- encoder_token_counts.append(num_tokens)
+ token_lora_mapping.extend([lora_id] * tower_tokens)
+ encoder_token_counts.append(tower_tokens)
+ connector_token_counts.append(connector_tokens)
if lora_id > 0:
lora_request = self.input_batch.lora_id_to_lora_request.get(lora_id)
@@ -3162,16 +3168,11 @@ class GPUModelRunner(
if (
mm_mapping is not None
and mm_mapping.connector
- and hasattr(self.model, "get_num_mm_connector_tokens")
+ and all(count is not None for count in connector_token_counts)
):
- post_op_counts = [
- self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined]
- for num_tokens in encoder_token_counts
- ]
-
connector_token_mapping = np.repeat(
np.array(prompt_lora_mapping, dtype=np.int32),
- np.array(post_op_counts, dtype=np.int32),
+ np.array(connector_token_counts, dtype=np.int32),
)
connector_mapping = LoRAMapping(
index_mapping=tuple(connector_token_mapping.tolist()),