From fe5145765f211c9b5abc446b42491f84895d70ba Mon Sep 17 00:00:00 2001 From: Tri Vo Date: Sat, 25 Jul 2026 17:06:14 +0700 Subject: [PATCH] [Core] Keep attention backends eligible for text-only serving of prefix-LM models (#48796) Signed-off-by: qtris123 Co-authored-by: Cyrus Leung --- tests/config/test_multimodal_config.py | 100 ++++++++++++++++++ tests/models/utils.py | 6 ++ vllm/config/model.py | 44 +++++++- .../model_arch_config_convertor.py | 22 ++-- 4 files changed, 165 insertions(+), 7 deletions(-) diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 5260d7a40f..befcc73b0b 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -1,10 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import MagicMock, patch + import pytest +from transformers import PretrainedConfig from vllm.config.model import ModelConfig from vllm.config.multimodal import MultiModalConfig +from vllm.transformers_utils.model_arch_config_convertor import ( + ModelArchConfigConvertorBase, +) from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -68,3 +74,97 @@ def test_mm_encoder_attn_dtype_hash_updates(tmp_path): ).compute_hash() assert base_hash != fp8_hash assert fp8_hash != fp8_static_hash + + +def _make_mm_prefix_model_config( + *, + language_model_only: bool = False, +) -> ModelConfig: + model_config = MagicMock(spec=ModelConfig) + model_config.multimodal_config = MultiModalConfig( + language_model_only=language_model_only + ) + # Bind real helper methods onto the mock. + model_config._supports_multimodal_for_mm_prefix = ( + ModelConfig._supports_multimodal_for_mm_prefix.__get__( + model_config, ModelConfig + ) + ) + return model_config + + +@pytest.mark.parametrize("supports_mm", [True, False]) +def test_supports_multimodal_for_mm_prefix_uses_registry(supports_mm: bool): + model_config = _make_mm_prefix_model_config() + + with patch( + "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs", + return_value=supports_mm, + ) as mocked: + assert model_config._supports_multimodal_for_mm_prefix() is supports_mm + mocked.assert_called_once_with(model_config) + + # Sticky cache — registry must not be consulted again. + with patch( + "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs", + side_effect=AssertionError("should use cache"), + ): + assert model_config._supports_multimodal_for_mm_prefix() is supports_mm + + +def test_supports_multimodal_for_mm_prefix_before_multimodal_config(): + model_config = _make_mm_prefix_model_config() + model_config.multimodal_config = None + + assert model_config._supports_multimodal_for_mm_prefix() is True + assert not hasattr(model_config, "_supports_multimodal_inputs_cached") + + +def test_language_model_only_disables_via_supports_multimodal_inputs(): + """language_model_only zeros all limits, so registry reports text-only.""" + model_config = _make_mm_prefix_model_config(language_model_only=True) + + with patch( + "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs", + return_value=False, + ): + assert model_config._supports_multimodal_for_mm_prefix() is False + + +def test_convertor_clears_mm_prefix_when_multimodal_disabled(): + hf_config = PretrainedConfig( + model_type="gemma3", + architectures=["Gemma3ForConditionalGeneration"], + ) + hf_config.is_mm_prefix_lm = True + convertor = ModelArchConfigConvertorBase(hf_config, hf_config) + + assert convertor.is_mm_prefix_lm(supports_multimodal=True) is True + assert convertor.is_mm_prefix_lm(supports_multimodal=False) is False + + enabled = convertor.convert(supports_multimodal=True) + disabled = convertor.convert(supports_multimodal=False) + assert enabled.is_mm_prefix_lm is True + assert disabled.is_mm_prefix_lm is False + + +def test_sticky_cache_survives_text_subconfig_regeneration(): + """with_hf_config deepcopies the cached decision onto text submodules.""" + model_config = _make_mm_prefix_model_config() + with patch( + "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs", + return_value=False, + ): + assert model_config._supports_multimodal_for_mm_prefix() is False + + # Simulate deepcopy onto a Gemma4ForCausalLM-like config that would + # otherwise fail registry lookup / return False incorrectly. + text_config = _make_mm_prefix_model_config() + text_config._supports_multimodal_inputs_cached = ( + model_config._supports_multimodal_inputs_cached + ) + with patch( + "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs", + side_effect=AssertionError("must not re-query registry"), + ): + assert text_config._supports_multimodal_for_mm_prefix() is False diff --git a/tests/models/utils.py b/tests/models/utils.py index 86fd80cd62..f938a70223 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -507,6 +507,12 @@ def dummy_hf_overrides( hf_config = _hf_config hf_text_config = text_config + # Keep architecture conversion on the default multimodal path; this + # helper only needs HF-derived fields, not deployment MM limits. + @staticmethod + def _supports_multimodal_for_mm_prefix() -> bool: + return True + model_arch_config = ModelConfig.get_model_arch_config(DummyConfig) # Only set MoE related config when the model has MoE layers. # Otherwise all models detected as MoE by _get_transformers_backend_cls. diff --git a/vllm/config/model.py b/vllm/config/model.py index a65d5d5a94..6b7825be08 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -758,6 +758,10 @@ class ModelConfig: "disable the cache with --mm-processor-cache-gb 0." ) + # Rebuild after multimodal_config exists so text-only mm_prefix + # clearing is applied (and cached for later with_hf_config calls). + self.model_arch_config = self.get_model_arch_config() + if self.disable_sliding_window: # Set after get_and_verify_max_len to ensure that max_model_len # can be correctly capped to sliding window size @@ -770,6 +774,42 @@ class ModelConfig: self._verify_cuda_graph() self._verify_bnb_config() + def _supports_multimodal_for_mm_prefix(self) -> bool: + """Whether multimodal inputs can still appear for this deployment. + + This runs more than once per config: once early in ``__post_init__`` + (before ``multimodal_config`` exists), again after it is created, and + then for every ``get_model_arch_config`` regeneration -- notably + ``with_hf_config``, which deep-copies this ``ModelConfig`` and swaps + ``hf_config`` for a text-only submodule (e.g. ``Gemma4ForCausalLM``). + + The result is cached for correctness, not just to save work: on the + ``with_hf_config`` copy the submodule architecture has no registered + multimodal processor, so re-querying the registry would raise and be + treated as text-only, wrongly clearing ``is_mm_prefix_lm`` even when a + vision modality is still enabled (e.g. ``image=0`` but video allowed). + The deep-copied cache preserves the top-level decision instead. + """ + cached = getattr(self, "_supports_multimodal_inputs_cached", None) + if cached is not None: + return cached + + if self.multimodal_config is None: + # Early call before multimodal init — do not clear mm_prefix yet. + return True + + from vllm.multimodal import MULTIMODAL_REGISTRY + + supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs(self) + self._supports_multimodal_inputs_cached = supports_mm + if not supports_mm: + logger.info_once( + "Disabled mm_prefix attention mode because multimodal inputs " + "are configuration-disabled. Attention backends without " + "mm_prefix support may now be selected." + ) + return supports_mm + def get_model_arch_config( self, ) -> ModelArchitectureConfig: @@ -777,7 +817,9 @@ class ModelConfig: self.hf_config.model_type, ModelArchConfigConvertorBase ) convertor = convertor_cls(self.hf_config, self.hf_text_config) - return convertor.convert() + return convertor.convert( + supports_multimodal=self._supports_multimodal_for_mm_prefix() + ) @field_validator("tokenizer", "max_model_len", mode="wrap") @classmethod diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 70bb4caa53..e62b29295f 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -300,8 +300,16 @@ class ModelArchConfigConvertorBase: ) return False - def is_mm_prefix_lm(self) -> bool: - """Whether to use bidirectional attention for mm positions.""" + def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool: + """Whether to use bidirectional attention for mm positions. + + ``supports_multimodal`` is False when the deployment is configuration- + disabled for multimodal inputs (text-only serving). In that case + mm_prefix is unnecessary and must stay off so attention backends + without ``supports_mm_prefix()`` remain eligible. + """ + if not supports_multimodal: + return False if hasattr(self.hf_config, "is_mm_prefix_lm"): return bool(self.hf_config.is_mm_prefix_lm) # fallback to list of known models @@ -358,7 +366,7 @@ class ModelArchConfigConvertorBase: derived_max_model_len = tmp_max_len return derived_max_model_len, max_len_key - def convert(self) -> ModelArchitectureConfig: + def convert(self, supports_multimodal: bool = True) -> ModelArchitectureConfig: model_arch_config = ModelArchitectureConfig( architectures=self.get_architectures(), model_type=self.hf_config.model_type, @@ -372,7 +380,7 @@ class ModelArchConfigConvertorBase: num_experts=self.get_num_experts(), quantization_config=self.get_quantization_config(), is_deepseek_mla=self.is_deepseek_mla(), - is_mm_prefix_lm=self.is_mm_prefix_lm(), + is_mm_prefix_lm=self.is_mm_prefix_lm(supports_multimodal), rswa_window=self.rswa_window(), derived_max_model_len_and_key=self.derive_max_model_len_and_key(), ) @@ -401,7 +409,7 @@ class CohereAsrModelArchConfigConvertor(ModelArchConfigConvertorBase): ) return enc_num_kv_heads - def is_mm_prefix_lm(self) -> bool: + def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool: return False @@ -584,7 +592,9 @@ class Gemma4MTPModelArchConfigConvertor(ModelArchConfigConvertorBase): class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase): - def is_mm_prefix_lm(self) -> bool: + def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool: + if not supports_multimodal: + return False return ( getattr(self.hf_text_config, "use_bidirectional_attention", None) == "vision"