diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index a79196ffbd..4287816b90 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -48,3 +48,4 @@ steps: - pytest -v -s -x lora/test_olmoe_tp.py - pytest -v -s -x lora/test_gptoss_tp.py - pytest -v -s -x lora/test_qwen35_densemodel_lora.py + - pytest -v -s -x lora/test_gemma4_tp.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 64c7c7062a..d7b587b6df 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -523,7 +523,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + IE+ | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ | | `Gemma3ForConditionalGeneration` | Gemma 3 | T + IE+ | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ | | `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | -| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I+ + V + A* | `google/gemma-4-E2B-it`, etc. | | ✅︎ | +| `Gemma4ForConditionalGeneration` | Gemma 4 | T + I+ + V + A* | `google/gemma-4-E2B-it`, etc. | ✅︎ | ✅︎ | | `Gemma4UnifiedForConditionalGeneration` | Gemma 4 Unified | T + I+ + V + A | `google/gemma-4-12B-it`, etc. | | ✅︎ | | `GLM4VForCausalLM`^ | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ | | `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + IE+ + VE+ | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ | diff --git a/tests/lora/conftest.py b/tests/lora/conftest.py index 899b3129d6..f14b488a17 100644 --- a/tests/lora/conftest.py +++ b/tests/lora/conftest.py @@ -253,6 +253,11 @@ def qwen3vl_vision_lora_files(): ) +@pytest.fixture(scope="session") +def gemma4_vision_lora_files(): + return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon") + + @pytest.fixture(scope="session") def qwen3_meowing_lora_files(): """Download Qwen3 Meow LoRA files once per test session.""" diff --git a/tests/lora/test_gemma4_tp.py b/tests/lora/test_gemma4_tp.py new file mode 100644 index 0000000000..efbee00d62 --- /dev/null +++ b/tests/lora/test_gemma4_tp.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# NOTE To avoid overloading the CI pipeline, this test script will not +# be triggered on CI and is primarily intended for local testing and verification. + +import vllm +from vllm.assets.image import ImageAsset +from vllm.lora.request import LoRARequest + +from ..utils import multi_gpu_test + +MODEL_PATH = "google/gemma-4-E2B-it" + +PROMPT_TEMPLATE = """<|turn>user +<|image|>What is in the image? +<|turn>model +""" + +TEST_IMAGES = [ + ImageAsset("stop_sign"), + ImageAsset("cherry_blossom"), +] + +EXPECTED_OUTPUTS_VISION = [ + "A red stop sign stands prominently in the foreground.", + "A majestic skyscraper stands tall, partially obscured by a vibrant " + "canopy of cherry blossoms, against a clear blue sky.", +] + + +def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None: + prompts = [ + { + "prompt": PROMPT_TEMPLATE, + "multi_modal_data": {"image": asset.pil_image}, + } + for asset in TEST_IMAGES + ] + sampling_params = vllm.SamplingParams(temperature=0, max_tokens=128) + outputs = llm.generate( + prompts, + sampling_params, + lora_request=LoRARequest(str(lora_id), lora_id, lora_path), + ) + + generated_texts = [output.outputs[0].text.strip() for output in outputs] + for generated, expected in zip(generated_texts, EXPECTED_OUTPUTS_VISION): + assert generated.startswith(expected), ( + f"Generated text {generated!r} does not match expected {expected!r}" + ) + + +def test_gemma4_lora(gemma4_vision_lora_files): + llm = vllm.LLM( + MODEL_PATH, + max_model_len=1024, + enable_lora=True, + enforce_eager=True, + max_loras=4, + trust_remote_code=True, + limit_mm_per_prompt={"image": 1}, + mm_processor_cache_gb=0, + enable_tower_connector_lora=True, + ) + + generate_and_test(llm, gemma4_vision_lora_files, lora_id=1) + generate_and_test(llm, gemma4_vision_lora_files, lora_id=2) + + +@multi_gpu_test(num_gpus=2) +def test_gemma4_lora_tp2(gemma4_vision_lora_files): + llm = vllm.LLM( + MODEL_PATH, + max_model_len=1024, + enforce_eager=True, + enable_lora=True, + max_loras=4, + trust_remote_code=True, + tensor_parallel_size=2, + limit_mm_per_prompt={"image": 1}, + mm_processor_cache_gb=0, + enable_tower_connector_lora=True, + ) + + generate_and_test(llm, gemma4_vision_lora_files, lora_id=1) + generate_and_test(llm, gemma4_vision_lora_files, lora_id=2) + + +@multi_gpu_test(num_gpus=4) +def test_gemma4_lora_tp4(gemma4_vision_lora_files): + llm = vllm.LLM( + MODEL_PATH, + max_model_len=1024, + enforce_eager=True, + enable_lora=True, + max_loras=4, + trust_remote_code=True, + tensor_parallel_size=4, + limit_mm_per_prompt={"image": 1}, + mm_processor_cache_gb=0, + enable_tower_connector_lora=True, + ) + + generate_and_test(llm, gemma4_vision_lora_files, lora_id=1) + generate_and_test(llm, gemma4_vision_lora_files, lora_id=2) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 2db024dc6b..06e3fe4fa7 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -380,9 +380,13 @@ def test_select_common_block_size_no_valid_option(): def test_set_active_mm_loras_builds_tower_and_connector_mappings(): model = Mock() - model.get_num_mm_encoder_tokens.side_effect = lambda num_embeds: num_embeds + 1 + model.get_mm_lora_token_counts.side_effect = ( + lambda *, modality, mm_kwargs, num_mm_embeds: ( + num_mm_embeds + 1, + num_mm_embeds + 11, + ) + ) model.get_mm_mapping.return_value = SimpleNamespace(connector=True) - model.get_num_mm_connector_tokens.side_effect = lambda num_tokens: num_tokens + 10 lora_manager = Mock() lora_manager.supports_tower_connector_lora.return_value = True diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index bd5e91cfa5..5db3cd4faa 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -247,8 +247,17 @@ class LoRAModelManager: mm_budget = MultiModalBudget(vllm_config, mm_registry) limit_per_prompt = max(mm_budget.mm_max_items_per_prompt.values()) - num_encoder_tokens = self.model.get_num_mm_encoder_tokens( - mm_budget.get_encoder_budget() + max_lora_tokens = mm_budget.get_encoder_budget() + lora_token_counts_by_modality = [ + self.model.get_mm_lora_token_counts( + modality=modality, + mm_kwargs=None, + num_mm_embeds=max_lora_tokens, + ) + for modality in mm_budget.mm_max_toks_per_item + ] + num_encoder_tokens = max( + tower_tokens for tower_tokens, _ in lora_token_counts_by_modality ) # Tower wrappers @@ -263,10 +272,15 @@ class LoRAModelManager: # Use wrapper for connector if present. if self.mm_mapping.connector: - if hasattr(self.model, "get_num_mm_connector_tokens"): - connector_tokens = self.model.get_num_mm_connector_tokens( - num_encoder_tokens - ) + connector_tokens = max( + ( + connector_tokens + for _, connector_tokens in lora_token_counts_by_modality + if connector_tokens is not None + ), + default=None, + ) + if connector_tokens is not None: connector_punica_wrapper = get_punica_wrapper( connector_tokens, max_batches=self.max_num_seqs * limit_per_prompt, diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 1ef236907f..1bfc0c0b7f 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -50,6 +50,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, VideoItem, ) @@ -1037,6 +1038,10 @@ class Gemma4ForConditionalGeneration( self.multimodal_config = multimodal_config self.model_dtype = vllm_config.model_config.dtype self.vllm_config = vllm_config + lora_config = vllm_config.lora_config + self._enable_mm_lora = bool( + lora_config is not None and lora_config.enable_tower_connector_lora + ) # Only quantize towers when the quant method supports their # dimensions. BNB/torchao handle arbitrary sizes; other methods @@ -1265,8 +1270,8 @@ class Gemma4ForConditionalGeneration( Groups images by patch count (resolution bucket) so each encoder call processes a uniform-shape batch with no - cross-resolution padding. Pooling and projection are then - applied over a single concatenated tensor for all images. + cross-resolution padding. With MM LoRA enabled, all images are + padded into one batch so the encoder call matches the tower mapping. """ pixel_values = image_input["pixel_values"] pixel_position_ids = image_input["pixel_position_ids"] @@ -1284,24 +1289,65 @@ class Gemma4ForConditionalGeneration( if isinstance(pixel_values, list) else pixel_values.shape[0] ) + pool_position_ids = pixel_position_ids - for idx in range(total_images): - pv = pixel_values[idx] - pp = pixel_position_ids[idx] - buckets.setdefault(pv.shape[0], []).append((idx, pv, pp)) + if self._enable_mm_lora: + max_soft_tokens = vision_cfg.default_output_length + mm_processor_kwargs = getattr( + getattr(self, "multimodal_config", None), + "mm_processor_kwargs", + None, + ) + if isinstance(mm_processor_kwargs, Mapping): + value, _ = _get_max_soft_tokens(mm_processor_kwargs) + if isinstance(value, int) and value in _SUPPORTED_SOFT_TOKENS: + max_soft_tokens = value + + max_patches = max_soft_tokens * pooling_k2 + padded_position_ids: list[torch.Tensor] = [] + for idx in range(total_images): + pv = pixel_values[idx] + pp = pixel_position_ids[idx] + num_patches = pv.shape[0] + if num_patches > max_patches: + raise ValueError( + f"Image {idx} has {num_patches} patches, which exceeds " + f"the MM LoRA patch limit of {max_patches}." + ) + + pad_len = max_patches - num_patches + pv = torch.cat( + (pv, pv.new_zeros((pad_len, *pv.shape[1:]))), + dim=0, + ) + pp = torch.cat( + (pp, pp.new_full((pad_len, *pp.shape[1:]), -1)), + dim=0, + ) + buckets.setdefault(max_patches, []).append((idx, pv, pp)) + padded_position_ids.append(pp) + pool_position_ids = padded_position_ids + else: + for idx in range(total_images): + pv = pixel_values[idx] + pp = pixel_position_ids[idx] + buckets.setdefault(pv.shape[0], []).append((idx, pv, pp)) # Encode each resolution bucket in memory-safe chunks. Re-read # free memory per bucket because the previous bucket's encoder # pass has already allocated activations we should account for. last_hidden_states_map: dict[int, torch.Tensor] = {} for patches, items in buckets.items(): - free, total = torch.accelerator.get_memory_info() - max_batch_size = min( - len(items), - self._encoder_chunk( - patches, free, total, vision_cfg.position_embedding_size - ), - ) + if self._enable_mm_lora: + max_batch_size = len(items) + else: + free, total = torch.accelerator.get_memory_info() + max_batch_size = min( + len(items), + self._encoder_chunk( + patches, free, total, vision_cfg.position_embedding_size + ), + ) for chunk_idx in range(0, len(items), max_batch_size): chunk_items = items[chunk_idx : chunk_idx + max_batch_size] @@ -1338,7 +1384,7 @@ class Gemma4ForConditionalGeneration( output_length = chunk_hidden.shape[0] // pooling_k2 single_hidden = chunk_hidden.unsqueeze(0) - single_pos_ids = pixel_position_ids[orig_idx].unsqueeze(0) + single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0) padding_positions = (single_pos_ids == -1).all(dim=-1) pooled_states, valid_mask = vt.pooler( @@ -2146,6 +2192,66 @@ class Gemma4ForConditionalGeneration( tower_model=tower_models, ) + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + if modality in ("image", "video"): + vision_config = self.config.vision_config + pooling_k2 = vision_config.pooling_kernel_size**2 + + if modality == "image": + pixel_values_key = "pixel_values" + max_soft_tokens = vision_config.default_output_length + mm_processor_kwargs = getattr( + getattr(self, "multimodal_config", None), + "mm_processor_kwargs", + None, + ) + if isinstance(mm_processor_kwargs, Mapping): + val, _ = _get_max_soft_tokens(mm_processor_kwargs) + if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS: + max_soft_tokens = val + else: + pixel_values_key = "pixel_values_videos" + max_soft_tokens = _VIDEO_MAX_SOFT_TOKENS + + tower_tokens = max_soft_tokens * pooling_k2 if modality == "image" else None + connector_tokens = num_mm_embeds + if tower_tokens is None and mm_kwargs is not None: + field = mm_kwargs.get(pixel_values_key) + if field is not None: + data = field.data + if isinstance(data, torch.Tensor) and data.ndim >= 2: + tower_tokens = int(math.prod(data.shape[:-1])) + + if tower_tokens is None: + min_soft_tokens = min(_SUPPORTED_SOFT_TOKENS) + tower_tokens = ( + math.ceil(num_mm_embeds / min_soft_tokens) + * max_soft_tokens + * pooling_k2 + ) + + if modality == "audio": + tower_tokens = num_mm_embeds + connector_tokens = num_mm_embeds + + if mm_kwargs is not None: + field = mm_kwargs.get("input_features_padded") + if field is not None: + data = field.data + if isinstance(data, torch.Tensor) and data.ndim >= 2: + batch_size = math.prod(data.shape[:-2]) + audio_tokens = batch_size * math.ceil(data.shape[-2] / 4) + tower_tokens = audio_tokens + connector_tokens = audio_tokens + + return tower_tokens, connector_tokens + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality == "image": diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 46d0ed30df..38fca287d5 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -48,7 +48,7 @@ if TYPE_CHECKING: from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc from vllm.model_executor.models.interfaces_base import VllmModel from vllm.model_executor.models.utils import WeightsMapper - from vllm.multimodal.inputs import MultiModalFeatureSpec + from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem from vllm.multimodal.registry import _ProcessorFactories from vllm.sequence import IntermediateTensors from vllm.tasks import ScoreType @@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol): """ ... + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: "MultiModalKwargsItem | None", + num_mm_embeds: int, + ) -> tuple[int, int | None]: + """ + Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings. + + MM LoRA uses these counts to build adapter mappings for the tower and + connector forwards. Models with multiple modalities can override this + when each modality has different encoder padding or pooling behavior. + """ + del modality, mm_kwargs + num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds) + num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens) + return ( + num_encoder_tokens, + num_connector_tokens if isinstance(num_connector_tokens, int) else None, + ) + @overload def embed_input_ids(self, input_ids: Tensor) -> Tensor: ... diff --git a/vllm/v1/worker/gpu/mm/lora.py b/vllm/v1/worker/gpu/mm/lora.py index 492914a74b..819f3b1cb3 100644 --- a/vllm/v1/worker/gpu/mm/lora.py +++ b/vllm/v1/worker/gpu/mm/lora.py @@ -29,6 +29,7 @@ def set_active_mm_loras( token_lora_mapping: list[int] = [] lora_requests = set() encoder_token_counts: list[int] = [] + connector_token_counts: list[int] = [] # iterate through images for req_id, encoder_input_ids in scheduled_encoder_inputs.items(): @@ -42,10 +43,17 @@ def set_active_mm_loras( # iterate through visual tokens for mm_input_id in encoder_input_ids: pos_info = mm_features[mm_input_id].mm_position - num_tokens = model.get_num_mm_encoder_tokens(pos_info.get_num_embeds()) + + tower_tokens, connector_tokens = model.get_mm_lora_token_counts( + modality=mm_features[mm_input_id].modality, + mm_kwargs=mm_features[mm_input_id].data, + num_mm_embeds=pos_info.get_num_embeds(), + ) + prompt_lora_mapping.append(lora_id) - token_lora_mapping.extend([lora_id] * num_tokens) - encoder_token_counts.append(num_tokens) + token_lora_mapping.extend([lora_id] * tower_tokens) + encoder_token_counts.append(tower_tokens) + connector_token_counts.append(connector_tokens) if lora_id > 0: lora_request = lora_state.lora_requests.get(req_id) @@ -69,19 +77,13 @@ def set_active_mm_loras( if ( mm_mapping is None or not mm_mapping.connector - or not hasattr(model, "get_num_mm_connector_tokens") + or not all(count is not None for count in connector_token_counts) ): return connector_token_mapping = np.repeat( np.array(prompt_lora_mapping, dtype=np.int32), - np.array( - [ - model.get_num_mm_connector_tokens(num_tokens) - for num_tokens in encoder_token_counts - ], - dtype=np.int32, - ), + np.array(connector_token_counts, dtype=np.int32), ) lora_manager.set_active_adapters( lora_requests, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index defaccacf6..e81913f928 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3123,18 +3123,24 @@ class GPUModelRunner( token_lora_mapping = [] lora_requests = set() encoder_token_counts = [] + connector_token_counts = [] - for req_id, pos_info in mm_lora_refs: + for (req_id, pos_info), (modality, mm_item) in zip( + mm_lora_refs, + mm_kwargs, + ): req_idx = self.input_batch.req_id_to_index[req_id] lora_id = int(self.input_batch.request_lora_mapping[req_idx]) - # Prefer pos_info.get_num_embeds to count precise MM embedding tokens. - num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined] - pos_info.get_num_embeds() + tower_tokens, connector_tokens = self.model.get_mm_lora_token_counts( # type: ignore[attr-defined] + modality=modality, + mm_kwargs=mm_item, + num_mm_embeds=pos_info.get_num_embeds(), ) prompt_lora_mapping.append(lora_id) - token_lora_mapping.extend([lora_id] * num_tokens) - encoder_token_counts.append(num_tokens) + token_lora_mapping.extend([lora_id] * tower_tokens) + encoder_token_counts.append(tower_tokens) + connector_token_counts.append(connector_tokens) if lora_id > 0: lora_request = self.input_batch.lora_id_to_lora_request.get(lora_id) @@ -3162,16 +3168,11 @@ class GPUModelRunner( if ( mm_mapping is not None and mm_mapping.connector - and hasattr(self.model, "get_num_mm_connector_tokens") + and all(count is not None for count in connector_token_counts) ): - post_op_counts = [ - self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined] - for num_tokens in encoder_token_counts - ] - connector_token_mapping = np.repeat( np.array(prompt_lora_mapping, dtype=np.int32), - np.array(post_op_counts, dtype=np.int32), + np.array(connector_token_counts, dtype=np.int32), ) connector_mapping = LoRAMapping( index_mapping=tuple(connector_token_mapping.tolist()),