diff --git a/tests/entrypoints/openai/chat_completion/test_chat_completion_with_prompt_embeds.py b/tests/entrypoints/openai/chat_completion/test_chat_completion_with_prompt_embeds.py index 1813d74798..3ab5185fe5 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_completion_with_prompt_embeds.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_completion_with_prompt_embeds.py @@ -14,6 +14,7 @@ import torch from openai import BadRequestError from tests.utils import VLLM_PATH, RemoteOpenAIServer +from vllm.platforms import current_platform MODEL_NAME = "facebook/opt-125m" CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja" @@ -41,7 +42,11 @@ def server_args() -> list[str]: @pytest.fixture(scope="module") -def server(server_args): +def server(server_args, request): + if current_platform.is_rocm(): + # Materialize HF embeddings before the server reserves ROCm VRAM. + request.getfixturevalue("prompt_embeds_b64") + request.getfixturevalue("aligned_content_and_embeds_b64") with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server: yield remote_server diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 4d900b5dde..338790dc69 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -72,7 +72,35 @@ class MultiModalProcessingInfo(BaseProcessingInfo): image_sizes=([height, width],), **mm_processor_kwargs ) image_tokens = mm_tokens["num_image_tokens"][0] - return image_tokens + return self._get_max_encoder_tokens(processor, mm_tokens) or image_tokens + + @staticmethod + def _get_mm_values(mm_tokens: object, key: str) -> object: + if isinstance(mm_tokens, Mapping): + return mm_tokens.get(key) + return getattr(mm_tokens, key, None) + + def _get_max_encoder_tokens( + self, processor: object, mm_tokens: object + ) -> int | None: + if "gemma3" not in processor.__class__.__name__.lower(): + return None + + vision_config = getattr(self.get_hf_config(), "vision_config", None) + image_size = getattr(vision_config, "image_size", None) + patch_size = getattr(vision_config, "patch_size", None) + if not image_size or not patch_size: + return None + + # Gemma3 pools each 64x64 SigLIP patch grid down to 256 image tokens. + # Profile the vision encoder against the pre-pooling patch-token count. + patches_per_image = (image_size // patch_size) ** 2 + num_image_patches = self._get_mm_values(mm_tokens, "num_image_patches") or [1] + if isinstance(num_image_patches, int): + max_image_patches = num_image_patches + else: + max_image_patches = max(num_image_patches) + return patches_per_image * int(max_image_patches) def get_max_image_size(self): return 10_000, 10_000 # hardcode for arbitrary very large size