[Core] Keep attention backends eligible for text-only serving of prefix-LM models (#48796)

Signed-off-by: qtris123 <voquangtri2021@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Tri Vo
2026-07-25 03:06:14 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent dbcc1cdd0a
commit fe5145765f
4 changed files with 165 additions and 7 deletions
+100
View File
@@ -1,10 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import pytest
from transformers import PretrainedConfig
from vllm.config.model import ModelConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -68,3 +74,97 @@ def test_mm_encoder_attn_dtype_hash_updates(tmp_path):
).compute_hash()
assert base_hash != fp8_hash
assert fp8_hash != fp8_static_hash
def _make_mm_prefix_model_config(
*,
language_model_only: bool = False,
) -> ModelConfig:
model_config = MagicMock(spec=ModelConfig)
model_config.multimodal_config = MultiModalConfig(
language_model_only=language_model_only
)
# Bind real helper methods onto the mock.
model_config._supports_multimodal_for_mm_prefix = (
ModelConfig._supports_multimodal_for_mm_prefix.__get__(
model_config, ModelConfig
)
)
return model_config
@pytest.mark.parametrize("supports_mm", [True, False])
def test_supports_multimodal_for_mm_prefix_uses_registry(supports_mm: bool):
model_config = _make_mm_prefix_model_config()
with patch(
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
return_value=supports_mm,
) as mocked:
assert model_config._supports_multimodal_for_mm_prefix() is supports_mm
mocked.assert_called_once_with(model_config)
# Sticky cache — registry must not be consulted again.
with patch(
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
side_effect=AssertionError("should use cache"),
):
assert model_config._supports_multimodal_for_mm_prefix() is supports_mm
def test_supports_multimodal_for_mm_prefix_before_multimodal_config():
model_config = _make_mm_prefix_model_config()
model_config.multimodal_config = None
assert model_config._supports_multimodal_for_mm_prefix() is True
assert not hasattr(model_config, "_supports_multimodal_inputs_cached")
def test_language_model_only_disables_via_supports_multimodal_inputs():
"""language_model_only zeros all limits, so registry reports text-only."""
model_config = _make_mm_prefix_model_config(language_model_only=True)
with patch(
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
return_value=False,
):
assert model_config._supports_multimodal_for_mm_prefix() is False
def test_convertor_clears_mm_prefix_when_multimodal_disabled():
hf_config = PretrainedConfig(
model_type="gemma3",
architectures=["Gemma3ForConditionalGeneration"],
)
hf_config.is_mm_prefix_lm = True
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
assert convertor.is_mm_prefix_lm(supports_multimodal=True) is True
assert convertor.is_mm_prefix_lm(supports_multimodal=False) is False
enabled = convertor.convert(supports_multimodal=True)
disabled = convertor.convert(supports_multimodal=False)
assert enabled.is_mm_prefix_lm is True
assert disabled.is_mm_prefix_lm is False
def test_sticky_cache_survives_text_subconfig_regeneration():
"""with_hf_config deepcopies the cached decision onto text submodules."""
model_config = _make_mm_prefix_model_config()
with patch(
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
return_value=False,
):
assert model_config._supports_multimodal_for_mm_prefix() is False
# Simulate deepcopy onto a Gemma4ForCausalLM-like config that would
# otherwise fail registry lookup / return False incorrectly.
text_config = _make_mm_prefix_model_config()
text_config._supports_multimodal_inputs_cached = (
model_config._supports_multimodal_inputs_cached
)
with patch(
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
side_effect=AssertionError("must not re-query registry"),
):
assert text_config._supports_multimodal_for_mm_prefix() is False
+6
View File
@@ -507,6 +507,12 @@ def dummy_hf_overrides(
hf_config = _hf_config
hf_text_config = text_config
# Keep architecture conversion on the default multimodal path; this
# helper only needs HF-derived fields, not deployment MM limits.
@staticmethod
def _supports_multimodal_for_mm_prefix() -> bool:
return True
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
# Only set MoE related config when the model has MoE layers.
# Otherwise all models detected as MoE by _get_transformers_backend_cls.