diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py index bd1f2bb867..24a30cae9d 100644 --- a/tests/models/multimodal/processing/test_gemma4.py +++ b/tests/models/multimodal/processing/test_gemma4.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +from PIL import Image as PILImage from vllm.multimodal import MULTIMODAL_REGISTRY @@ -66,6 +67,65 @@ def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens( ) +@pytest.mark.parametrize( + ("mm_processor_kwargs", "expected_image_tokens"), + [ + ({}, 280), + ({"max_soft_tokens": 70}, 70), + ({"max_soft_tokens": 280}, 280), + ({"max_soft_tokens": 1120}, 1120), + ({"images_kwargs": {"max_soft_tokens": 560}}, 560), + ({"images_kwargs": None}, 280), + ({"images_kwargs": "not-a-dict"}, 280), + ], +) +@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID]) +def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens( + model_id: str, + mm_processor_kwargs: dict[str, object], + expected_image_tokens: int, +): + ctx = build_model_context( + model_id, + mm_processor_kwargs=mm_processor_kwargs, + limit_mm_per_prompt={"image": 1, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + tokens = processor.info.get_mm_max_tokens_per_item( + seq_len=ctx.model_config.max_model_len, + mm_counts={"image": 1, "video": 1}, + ) + + assert tokens is not None + assert tokens["image"] == expected_image_tokens + assert tokens["video"] == 32 * (70 + 2 + 6) + + +@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID]) +def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str): + ctx = build_model_context( + model_id, + mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}}, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + image = PILImage.new("RGB", (1000, 1000), color="white") + image_size = image.size + mm_items = processor.info.parse_mm_data({"image": image}) + + prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0] + replacement = prompt_update.resolve(0).content.full + expected = processor.info.get_image_repl( + image_width=image_size[0], + image_height=image_size[1], + processor=processor.info.get_hf_processor(), + max_soft_tokens=560, + ).full + + assert replacement == expected + + @pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID]) def test_limit_mm_per_prompt( image_assets: ImageTestAssets, diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 9b2c54e273..029f4ff9bf 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -81,10 +81,26 @@ from .utils import ( logger = init_logger(__name__) # Video constants — match transformers Gemma4VideoProcessor defaults. +_SUPPORTED_SOFT_TOKENS = (70, 140, 280, 560, 1120) _VIDEO_MAX_SOFT_TOKENS = 70 # soft tokens per video frame (vs 280 for images) _VIDEO_MAX_FRAMES = 32 # max sampled frames per video +def _get_max_soft_tokens( + merged_kwargs: Mapping[str, object], +) -> tuple[object | None, bool]: + """Return configured image max_soft_tokens and whether it is top-level.""" + val = merged_kwargs.get("max_soft_tokens") + if val is not None: + return val, True + + images_kwargs = merged_kwargs.get("images_kwargs") + if isinstance(images_kwargs, Mapping): + return images_kwargs.get("max_soft_tokens"), False + + return None, False + + # --------------------------------------------------------------------------- # Input schema # --------------------------------------------------------------------------- @@ -216,10 +232,14 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): self, seq_len: int, mm_counts: Mapping[str, int] ) -> Mapping[str, int] | None: config = self.get_hf_config() - # Upper bound: the pooler outputs default_output_length slots - # per image (280). After padding is stripped the actual count - # is ≤ this value, but vLLM needs the max for memory planning. + # Upper bound: the pooler outputs max_soft_tokens slots per image. + # After padding is stripped the actual count is ≤ this value, but + # vLLM needs the max for memory planning. tokens_per_image = config.vision_config.default_output_length + merged_kwargs = self.ctx.get_merged_mm_kwargs({}) + val, _ = _get_max_soft_tokens(merged_kwargs) + if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS: + tokens_per_image = val tokens: dict[str, int] = {"image": tokens_per_image} if config.audio_config is not None: # Audio max tokens from the processor's audio_seq_length. @@ -492,13 +512,8 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): mm_kwargs: Mapping[str, object], tok_kwargs: Mapping[str, object], ) -> BatchFeature: - # Validate max_soft_tokens early and exit cleanly on bad values. - _SUPPORTED_SOFT_TOKENS = (70, 140, 280, 560, 1120) - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) - val = merged_kwargs.get("max_soft_tokens") - if val is None: - val = merged_kwargs.get("images_kwargs", {}).get("max_soft_tokens") + val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs) if val is not None and val not in _SUPPORTED_SOFT_TOKENS: raise ValueError( @@ -645,7 +660,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): # HF side (Gemma4ProcessorKwargs.images_kwargs) so that # _merge_kwargs routes max_soft_tokens into images_kwargs. patched_mm_kwargs = dict(mm_kwargs) - if val is not None: + if val is not None and is_top_level_max_soft_tokens: patched_mm_kwargs["max_soft_tokens"] = val processed_outputs = super()._call_hf_processor( @@ -755,7 +770,12 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): merged_kwargs = self.info.ctx.get_merged_mm_kwargs( hf_processor_mm_kwargs, ) - max_soft_tokens = merged_kwargs.get("max_soft_tokens") + val, _ = _get_max_soft_tokens(merged_kwargs) + max_soft_tokens = ( + val + if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS + else None + ) return self.info.get_image_repl( image_width=image_size.width, image_height=image_size.height,