[LoRA][Gemma4] Support vision tower LoRA (#42662)

Signed-off-by: bk-201 <joy25810@foxmail.com>
Signed-off-by: linitra24 <Joy25810@foxmail.com>
Signed-off-by: linitra24 <joy25810@gmail.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: linitra24 <joy25810@gmail.com>
This commit is contained in:
linitra24
2026-08-13 22:07:43 +08:00
committed by GitHub
co-authored by Jee Jee Li linitra24
parent 8e1131e1d1
commit 95c9144424
10 changed files with 309 additions and 48 deletions
+1
View File
@@ -48,3 +48,4 @@ steps:
- pytest -v -s -x lora/test_olmoe_tp.py
- pytest -v -s -x lora/test_gptoss_tp.py
- pytest -v -s -x lora/test_qwen35_densemodel_lora.py
- pytest -v -s -x lora/test_gemma4_tp.py
+1 -1
View File
@@ -523,7 +523,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
| `Gemma3ForConditionalGeneration` | Gemma 3 | T + I<sup>E+</sup> | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ |
| `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I<sup>+</sup> + V + A<sup>*</sup> | `google/gemma-4-E2B-it`, etc. | | ✅︎ |
| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I<sup>+</sup> + V + A<sup>*</sup> | `google/gemma-4-E2B-it`, etc. | ✅︎ | ✅︎ |
| `Gemma4UnifiedForConditionalGeneration` | Gemma 4 Unified | T + I<sup>+</sup> + V + A | `google/gemma-4-12B-it`, etc. | | ✅︎ |
| `GLM4VForCausalLM`<sup>^</sup> | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ |
| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ |
+5
View File
@@ -253,6 +253,11 @@ def qwen3vl_vision_lora_files():
)
@pytest.fixture(scope="session")
def gemma4_vision_lora_files():
return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon")
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 Meow LoRA files once per test session."""
+106
View File
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# NOTE To avoid overloading the CI pipeline, this test script will not
# be triggered on CI and is primarily intended for local testing and verification.
import vllm
from vllm.assets.image import ImageAsset
from vllm.lora.request import LoRARequest
from ..utils import multi_gpu_test
MODEL_PATH = "google/gemma-4-E2B-it"
PROMPT_TEMPLATE = """<bos><|turn>user
<|image|>What is in the image?<turn|>
<|turn>model
"""
TEST_IMAGES = [
ImageAsset("stop_sign"),
ImageAsset("cherry_blossom"),
]
EXPECTED_OUTPUTS_VISION = [
"A red stop sign stands prominently in the foreground.",
"A majestic skyscraper stands tall, partially obscured by a vibrant "
"canopy of cherry blossoms, against a clear blue sky.",
]
def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
prompts = [
{
"prompt": PROMPT_TEMPLATE,
"multi_modal_data": {"image": asset.pil_image},
}
for asset in TEST_IMAGES
]
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=128)
outputs = llm.generate(
prompts,
sampling_params,
lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
)
generated_texts = [output.outputs[0].text.strip() for output in outputs]
for generated, expected in zip(generated_texts, EXPECTED_OUTPUTS_VISION):
assert generated.startswith(expected), (
f"Generated text {generated!r} does not match expected {expected!r}"
)
def test_gemma4_lora(gemma4_vision_lora_files):
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enable_lora=True,
enforce_eager=True,
max_loras=4,
trust_remote_code=True,
limit_mm_per_prompt={"image": 1},
mm_processor_cache_gb=0,
enable_tower_connector_lora=True,
)
generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
@multi_gpu_test(num_gpus=2)
def test_gemma4_lora_tp2(gemma4_vision_lora_files):
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enforce_eager=True,
enable_lora=True,
max_loras=4,
trust_remote_code=True,
tensor_parallel_size=2,
limit_mm_per_prompt={"image": 1},
mm_processor_cache_gb=0,
enable_tower_connector_lora=True,
)
generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
@multi_gpu_test(num_gpus=4)
def test_gemma4_lora_tp4(gemma4_vision_lora_files):
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enforce_eager=True,
enable_lora=True,
max_loras=4,
trust_remote_code=True,
tensor_parallel_size=4,
limit_mm_per_prompt={"image": 1},
mm_processor_cache_gb=0,
enable_tower_connector_lora=True,
)
generate_and_test(llm, gemma4_vision_lora_files, lora_id=1)
generate_and_test(llm, gemma4_vision_lora_files, lora_id=2)
+6 -2
View File
@@ -380,9 +380,13 @@ def test_select_common_block_size_no_valid_option():
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
model = Mock()
model.get_num_mm_encoder_tokens.side_effect = lambda num_embeds: num_embeds + 1
model.get_mm_lora_token_counts.side_effect = (
lambda *, modality, mm_kwargs, num_mm_embeds: (
num_mm_embeds + 1,
num_mm_embeds + 11,
)
)
model.get_mm_mapping.return_value = SimpleNamespace(connector=True)
model.get_num_mm_connector_tokens.side_effect = lambda num_tokens: num_tokens + 10
lora_manager = Mock()
lora_manager.supports_tower_connector_lora.return_value = True
+20 -6
View File
@@ -247,8 +247,17 @@ class LoRAModelManager:
mm_budget = MultiModalBudget(vllm_config, mm_registry)
limit_per_prompt = max(mm_budget.mm_max_items_per_prompt.values())
num_encoder_tokens = self.model.get_num_mm_encoder_tokens(
mm_budget.get_encoder_budget()
max_lora_tokens = mm_budget.get_encoder_budget()
lora_token_counts_by_modality = [
self.model.get_mm_lora_token_counts(
modality=modality,
mm_kwargs=None,
num_mm_embeds=max_lora_tokens,
)
for modality in mm_budget.mm_max_toks_per_item
]
num_encoder_tokens = max(
tower_tokens for tower_tokens, _ in lora_token_counts_by_modality
)
# Tower wrappers
@@ -263,10 +272,15 @@ class LoRAModelManager:
# Use wrapper for connector if present.
if self.mm_mapping.connector:
if hasattr(self.model, "get_num_mm_connector_tokens"):
connector_tokens = self.model.get_num_mm_connector_tokens(
num_encoder_tokens
)
connector_tokens = max(
(
connector_tokens
for _, connector_tokens in lora_token_counts_by_modality
if connector_tokens is not None
),
default=None,
)
if connector_tokens is not None:
connector_punica_wrapper = get_punica_wrapper(
connector_tokens,
max_batches=self.max_num_seqs * limit_per_prompt,
+120 -14
View File
@@ -50,6 +50,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -1037,6 +1038,10 @@ class Gemma4ForConditionalGeneration(
self.multimodal_config = multimodal_config
self.model_dtype = vllm_config.model_config.dtype
self.vllm_config = vllm_config
lora_config = vllm_config.lora_config
self._enable_mm_lora = bool(
lora_config is not None and lora_config.enable_tower_connector_lora
)
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
@@ -1265,8 +1270,8 @@ class Gemma4ForConditionalGeneration(
Groups images by patch count (resolution bucket) so each
encoder call processes a uniform-shape batch with no
cross-resolution padding. Pooling and projection are then
applied over a single concatenated tensor for all images.
cross-resolution padding. With MM LoRA enabled, all images are
padded into one batch so the encoder call matches the tower mapping.
"""
pixel_values = image_input["pixel_values"]
pixel_position_ids = image_input["pixel_position_ids"]
@@ -1284,24 +1289,65 @@ class Gemma4ForConditionalGeneration(
if isinstance(pixel_values, list)
else pixel_values.shape[0]
)
pool_position_ids = pixel_position_ids
for idx in range(total_images):
pv = pixel_values[idx]
pp = pixel_position_ids[idx]
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
if self._enable_mm_lora:
max_soft_tokens = vision_cfg.default_output_length
mm_processor_kwargs = getattr(
getattr(self, "multimodal_config", None),
"mm_processor_kwargs",
None,
)
if isinstance(mm_processor_kwargs, Mapping):
value, _ = _get_max_soft_tokens(mm_processor_kwargs)
if isinstance(value, int) and value in _SUPPORTED_SOFT_TOKENS:
max_soft_tokens = value
max_patches = max_soft_tokens * pooling_k2
padded_position_ids: list[torch.Tensor] = []
for idx in range(total_images):
pv = pixel_values[idx]
pp = pixel_position_ids[idx]
num_patches = pv.shape[0]
if num_patches > max_patches:
raise ValueError(
f"Image {idx} has {num_patches} patches, which exceeds "
f"the MM LoRA patch limit of {max_patches}."
)
pad_len = max_patches - num_patches
pv = torch.cat(
(pv, pv.new_zeros((pad_len, *pv.shape[1:]))),
dim=0,
)
pp = torch.cat(
(pp, pp.new_full((pad_len, *pp.shape[1:]), -1)),
dim=0,
)
buckets.setdefault(max_patches, []).append((idx, pv, pp))
padded_position_ids.append(pp)
pool_position_ids = padded_position_ids
else:
for idx in range(total_images):
pv = pixel_values[idx]
pp = pixel_position_ids[idx]
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
# Encode each resolution bucket in memory-safe chunks. Re-read
# free memory per bucket because the previous bucket's encoder
# pass has already allocated activations we should account for.
last_hidden_states_map: dict[int, torch.Tensor] = {}
for patches, items in buckets.items():
free, total = torch.accelerator.get_memory_info()
max_batch_size = min(
len(items),
self._encoder_chunk(
patches, free, total, vision_cfg.position_embedding_size
),
)
if self._enable_mm_lora:
max_batch_size = len(items)
else:
free, total = torch.accelerator.get_memory_info()
max_batch_size = min(
len(items),
self._encoder_chunk(
patches, free, total, vision_cfg.position_embedding_size
),
)
for chunk_idx in range(0, len(items), max_batch_size):
chunk_items = items[chunk_idx : chunk_idx + max_batch_size]
@@ -1338,7 +1384,7 @@ class Gemma4ForConditionalGeneration(
output_length = chunk_hidden.shape[0] // pooling_k2
single_hidden = chunk_hidden.unsqueeze(0)
single_pos_ids = pixel_position_ids[orig_idx].unsqueeze(0)
single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
padding_positions = (single_pos_ids == -1).all(dim=-1)
pooled_states, valid_mask = vt.pooler(
@@ -2146,6 +2192,66 @@ class Gemma4ForConditionalGeneration(
tower_model=tower_models,
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
if modality in ("image", "video"):
vision_config = self.config.vision_config
pooling_k2 = vision_config.pooling_kernel_size**2
if modality == "image":
pixel_values_key = "pixel_values"
max_soft_tokens = vision_config.default_output_length
mm_processor_kwargs = getattr(
getattr(self, "multimodal_config", None),
"mm_processor_kwargs",
None,
)
if isinstance(mm_processor_kwargs, Mapping):
val, _ = _get_max_soft_tokens(mm_processor_kwargs)
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
max_soft_tokens = val
else:
pixel_values_key = "pixel_values_videos"
max_soft_tokens = _VIDEO_MAX_SOFT_TOKENS
tower_tokens = max_soft_tokens * pooling_k2 if modality == "image" else None
connector_tokens = num_mm_embeds
if tower_tokens is None and mm_kwargs is not None:
field = mm_kwargs.get(pixel_values_key)
if field is not None:
data = field.data
if isinstance(data, torch.Tensor) and data.ndim >= 2:
tower_tokens = int(math.prod(data.shape[:-1]))
if tower_tokens is None:
min_soft_tokens = min(_SUPPORTED_SOFT_TOKENS)
tower_tokens = (
math.ceil(num_mm_embeds / min_soft_tokens)
* max_soft_tokens
* pooling_k2
)
if modality == "audio":
tower_tokens = num_mm_embeds
connector_tokens = num_mm_embeds
if mm_kwargs is not None:
field = mm_kwargs.get("input_features_padded")
if field is not None:
data = field.data
if isinstance(data, torch.Tensor) and data.ndim >= 2:
batch_size = math.prod(data.shape[:-2])
audio_tokens = batch_size * math.ceil(data.shape[-2] / 4)
tower_tokens = audio_tokens
connector_tokens = audio_tokens
return tower_tokens, connector_tokens
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality == "image":
+23 -1
View File
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""
...
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: "MultiModalKwargsItem | None",
num_mm_embeds: int,
) -> tuple[int, int | None]:
"""
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
MM LoRA uses these counts to build adapter mappings for the tower and
connector forwards. Models with multiple modalities can override this
when each modality has different encoder padding or pooling behavior.
"""
del modality, mm_kwargs
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
return (
num_encoder_tokens,
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
)
@overload
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
+13 -11
View File
@@ -29,6 +29,7 @@ def set_active_mm_loras(
token_lora_mapping: list[int] = []
lora_requests = set()
encoder_token_counts: list[int] = []
connector_token_counts: list[int] = []
# iterate through images
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
@@ -42,10 +43,17 @@ def set_active_mm_loras(
# iterate through visual tokens
for mm_input_id in encoder_input_ids:
pos_info = mm_features[mm_input_id].mm_position
num_tokens = model.get_num_mm_encoder_tokens(pos_info.get_num_embeds())
tower_tokens, connector_tokens = model.get_mm_lora_token_counts(
modality=mm_features[mm_input_id].modality,
mm_kwargs=mm_features[mm_input_id].data,
num_mm_embeds=pos_info.get_num_embeds(),
)
prompt_lora_mapping.append(lora_id)
token_lora_mapping.extend([lora_id] * num_tokens)
encoder_token_counts.append(num_tokens)
token_lora_mapping.extend([lora_id] * tower_tokens)
encoder_token_counts.append(tower_tokens)
connector_token_counts.append(connector_tokens)
if lora_id > 0:
lora_request = lora_state.lora_requests.get(req_id)
@@ -69,19 +77,13 @@ def set_active_mm_loras(
if (
mm_mapping is None
or not mm_mapping.connector
or not hasattr(model, "get_num_mm_connector_tokens")
or not all(count is not None for count in connector_token_counts)
):
return
connector_token_mapping = np.repeat(
np.array(prompt_lora_mapping, dtype=np.int32),
np.array(
[
model.get_num_mm_connector_tokens(num_tokens)
for num_tokens in encoder_token_counts
],
dtype=np.int32,
),
np.array(connector_token_counts, dtype=np.int32),
)
lora_manager.set_active_adapters(
lora_requests,
+14 -13
View File
@@ -3123,18 +3123,24 @@ class GPUModelRunner(
token_lora_mapping = []
lora_requests = set()
encoder_token_counts = []
connector_token_counts = []
for req_id, pos_info in mm_lora_refs:
for (req_id, pos_info), (modality, mm_item) in zip(
mm_lora_refs,
mm_kwargs,
):
req_idx = self.input_batch.req_id_to_index[req_id]
lora_id = int(self.input_batch.request_lora_mapping[req_idx])
# Prefer pos_info.get_num_embeds to count precise MM embedding tokens.
num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]
pos_info.get_num_embeds()
tower_tokens, connector_tokens = self.model.get_mm_lora_token_counts( # type: ignore[attr-defined]
modality=modality,
mm_kwargs=mm_item,
num_mm_embeds=pos_info.get_num_embeds(),
)
prompt_lora_mapping.append(lora_id)
token_lora_mapping.extend([lora_id] * num_tokens)
encoder_token_counts.append(num_tokens)
token_lora_mapping.extend([lora_id] * tower_tokens)
encoder_token_counts.append(tower_tokens)
connector_token_counts.append(connector_tokens)
if lora_id > 0:
lora_request = self.input_batch.lora_id_to_lora_request.get(lora_id)
@@ -3162,16 +3168,11 @@ class GPUModelRunner(
if (
mm_mapping is not None
and mm_mapping.connector
and hasattr(self.model, "get_num_mm_connector_tokens")
and all(count is not None for count in connector_token_counts)
):
post_op_counts = [
self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined]
for num_tokens in encoder_token_counts
]
connector_token_mapping = np.repeat(
np.array(prompt_lora_mapping, dtype=np.int32),
np.array(post_op_counts, dtype=np.int32),
np.array(connector_token_counts, dtype=np.int32),
)
connector_mapping = LoRAMapping(
index_mapping=tuple(connector_token_mapping.tolist()),